diff --git a/buildSrc/src/main/java/org/jastadd/relast/plugin/RelastTest.java b/buildSrc/src/main/java/org/jastadd/relast/plugin/RelastTest.java
index 4c84a86b16458cf3f94eb25806a528452a8ec1af..2193eab52b6ede597fd6458cefd7dec592e5737f 100644
--- a/buildSrc/src/main/java/org/jastadd/relast/plugin/RelastTest.java
+++ b/buildSrc/src/main/java/org/jastadd/relast/plugin/RelastTest.java
@@ -9,9 +9,7 @@ import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.TaskAction;
 
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.nio.file.*;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -165,22 +163,23 @@ public class RelastTest extends DefaultTask {
     return option != null && !option.isEmpty();
   }
 
-  private String[] genSuffixes = {".ast", ".jadd", "RefResolver.jadd", "ResolverStubs.jrag", "Serializer.jadd"};
+  private final String[] genSuffixes = {".ast", ".jadd", "RefResolver.jadd", "ResolverStubs.jrag", "Serializer.jadd"};
 
   @TaskAction
   void runTest() throws IOException {
     setGroup("verification");
     setDescription("Runs a relast test");
     Project project = getProject();
+    String absoluteProjectPath = project.getProjectDir().getAbsolutePath();
     if (isVerbose()) {
-      System.out.println("Running relast test");
+      System.out.println("Running relast test in " + project.getDisplayName());
       System.out.println("relast files: " + getRelastFiles());
       System.out.println("Deleting files");
     }
     // first, delete generated files
-    List<String> genFiles = new ArrayList<>();
+    List<Path> genFiles = new ArrayList<>();
     for (String suffix : genSuffixes) {
-      genFiles.add(getGrammarName() + suffix);
+      genFiles.add(Paths.get(absoluteProjectPath, getGrammarName() + suffix));
     }
     if (isVerbose()) {
       System.out.println("gen files: " + genFiles);
@@ -235,14 +234,14 @@ public class RelastTest extends DefaultTask {
           System.out.println("Start relast with args: " + args);
         }
         javaExecSpec.args(args);
-      });
+      }).assertNormalExitValue();
     });
     if (isRunJastAdd()) {
       if (isVerbose()) {
         System.out.println("Compile with JastAdd");
       }
       // check which files were actually generated
-      genFiles.removeIf(s -> !Paths.get(s).toFile().exists());
+      genFiles.removeIf(this::verboseFileNotExists);
       // finally, compile generated files
       project.getPlugins().withType(JavaPlugin.class, javaPlugin -> {
         SourceSetContainer sourceSets = (SourceSetContainer) project.getProperties().get("sourceSets");
@@ -267,9 +266,30 @@ public class RelastTest extends DefaultTask {
     }
   }
 
+  private String fileExtension(String filename) {
+    int indexOfDot = filename.lastIndexOf('.');
+    return indexOfDot == -1 ? filename : filename.substring(indexOfDot + 1);
+  }
+
+  private boolean verboseFileNotExists(Path path) {
+    boolean fileDoesNotExist = !Files.exists(path);
+    if (fileDoesNotExist && isVerbose()) {
+      System.out.println("Do not include " + path);
+    }
+    return fileDoesNotExist;
+  }
+
   private void createDirectory(Path path) throws IOException {
+    if (Files.exists(path) && Files.isDirectory(path)) {
+      return;
+    }
+    if (isVerbose()) {
+      System.out.println("Creating " + path.toAbsolutePath());
+    }
     try {
       Files.createDirectories(path);
+    } catch (FileAlreadyExistsException e) {
+      System.err.println("Skipping creation of already existing " + path);
     } catch (IOException e) {
       System.err.println("Could not create output directory " + path);
       throw e;
diff --git a/ros2rag.tests/build.gradle b/ros2rag.tests/build.gradle
index f5cf3a7c17cf525412866197429bcdcbffe963ec..5ed86502d99e106b73381a7ab1d67bc5725544c7 100644
--- a/ros2rag.tests/build.gradle
+++ b/ros2rag.tests/build.gradle
@@ -98,3 +98,7 @@ task compileExampleTest(type: RelastTest) {
 
 test.dependsOn compileExampleTest
 compileExampleTest.dependsOn preprocessExampleTest
+
+clean {
+    delete 'src/test/02-after-ros2rag/example/', 'src/test/03-after-relast/example/'
+}