diff --git a/cmd/transpile/testdata/mkcategory/mkcategory b/cmd/transpile/testdata/mkcategory/mkcategory
new file mode 100644
index 0000000..04e5eb2
--- /dev/null
+++ b/cmd/transpile/testdata/mkcategory/mkcategory
@@ -0,0 +1,49 @@
+#! /bin/sh -e
+
+fail () {
+    echo "ERROR: $*" 1>&2
+    exit 1
+}
+
+case $1:$2:$3 in
+    inventory::)
+        cat <<EOT
+{
+    "Puzzles": [1, 2, 3,
+4, 5]
+}
+EOT
+        ;;
+    puzzle:1:)
+        cat <<EOT
+{
+    "Answers": ["answer1.0"],
+    "Pre": {
+        "Authors": ["author1.0"],
+        "Body": "<h1>moo.</h1>"
+    }
+}
+EOT
+        ;;
+    puzzle:*)
+        fail "No such puzzle: $2"
+        ;;
+    file:1:moo.txt)
+        echo "Moo."
+        ;;
+    file:*:*)
+        fail "No such file: $2"
+        ;;
+    answer:1:answer1.0)
+        echo -n '{"Correct":true}'
+        ;;
+    answer:1:*)
+        echo '{"Correct":false}'
+        ;;
+    answer:*:*)
+        fail "Fail answer"
+        ;;
+    *)
+        fail "What is $1" 1>&2
+        ;;
+esac
diff --git a/pkg/transpile/category.go b/pkg/transpile/category.go
index 915a872..8a42d1c 100644
--- a/pkg/transpile/category.go
+++ b/pkg/transpile/category.go
@@ -9,6 +9,7 @@ import (
 	"os/exec"
 	"path"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/spf13/afero"
@@ -133,7 +134,12 @@ func (c FsCommandCategory) run(command string, args ...string) ([]byte, error) {
 	cmdargs := append([]string{command}, args...)
 	cmd := exec.CommandContext(ctx, "./"+path.Base(c.command), cmdargs...)
 	cmd.Dir = path.Dir(c.command)
-	return cmd.Output()
+	out, err := cmd.Output()
+	if err, ok := err.(*exec.ExitError); ok {
+		stderr := strings.TrimSpace(string(err.Stderr))
+		return nil, fmt.Errorf("%s (%s)", stderr, err.String())
+	}
+	return out, err
 }
 
 // Inventory returns a list of point values for this category.
diff --git a/pkg/transpile/category_test.go b/pkg/transpile/category_test.go
index 80eac5a..1407a52 100644
--- a/pkg/transpile/category_test.go
+++ b/pkg/transpile/category_test.go
@@ -3,6 +3,8 @@ package transpile
 import (
 	"bytes"
 	"io"
+	"os/exec"
+	"strings"
 	"testing"
 
 	"github.com/spf13/afero"
@@ -105,6 +107,23 @@ func TestOsFsCategory(t *testing.T) {
 		t.Error("File shouldn't exist")
 	}
 
+	if r, err := generated.Open(1, "cow.txt"); err != nil {
+		if e, ok := err.(*exec.ExitError); ok {
+			t.Error(err, string(e.Stderr))
+		} else {
+			t.Error(err)
+		}
+	} else {
+		defer r.Close()
+		buf := new(bytes.Buffer)
+		if _, err := io.Copy(buf, r); err != nil {
+			t.Error(err)
+		}
+		if !strings.Contains(buf.String(), "moo.") {
+			t.Errorf("Wrong body: %#v", buf.String())
+		}
+	}
+
 	if !generated.Answer(1, "answer1.0") {
 		t.Error("Correct answer failed")
 	}
diff --git a/pkg/transpile/puzzle.go b/pkg/transpile/puzzle.go
index 4d9c512..7e75a7b 100644
--- a/pkg/transpile/puzzle.go
+++ b/pkg/transpile/puzzle.go
@@ -383,7 +383,12 @@ func (fp FsCommandPuzzle) run(command string, args ...string) ([]byte, error) {
 	cmdargs := append([]string{command}, args...)
 	cmd := exec.CommandContext(ctx, "./"+path.Base(fp.command), cmdargs...)
 	cmd.Dir = path.Dir(fp.command)
-	return cmd.Output()
+	out, err := cmd.Output()
+	if err, ok := err.(*exec.ExitError); ok {
+		stderr := strings.TrimSpace(string(err.Stderr))
+		return nil, fmt.Errorf("%s (%s)", stderr, err.String())
+	}
+	return out, err
 }
 
 // Puzzle returns a Puzzle struct for the current puzzle.
diff --git a/pkg/transpile/testdata/generated/cow.txt b/pkg/transpile/testdata/generated/cow.txt
new file mode 100644
index 0000000..b4e62b9
--- /dev/null
+++ b/pkg/transpile/testdata/generated/cow.txt
@@ -0,0 +1,8 @@
+ ______
+< moo. >
+ ------
+        \   ^__^
+         \  (oo)\_______
+            (__)\       )\/\
+                ||----w |
+                ||     ||
diff --git a/pkg/transpile/testdata/generated/mkcategory b/pkg/transpile/testdata/generated/mkcategory
index 04e5eb2..caae745 100755
--- a/pkg/transpile/testdata/generated/mkcategory
+++ b/pkg/transpile/testdata/generated/mkcategory
@@ -32,7 +32,7 @@ EOT
         echo "Moo."
         ;;
     file:*:*)
-        fail "No such file: $2"
+        cat "$3" || exit 1
         ;;
     answer:1:answer1.0)
         echo -n '{"Correct":true}'