More elaborated test case for errors.

- expected error output does not have to be in actual order, and can contain empty lines and comments which both will be ignored

src/test/jastadd/errors/Inheritance.expected

 Errors:
 $FILENAME Line 4, column 12: Token 'X' is an invalid redefinition for type 'B3', conflicts with supertype 'A'
-$FILENAME Line 7, column 5: Role 'X' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 4, column 23: Token 'MyName' is an invalid redefinition for type 'B3', conflicts with supertype 'A'
+
+$FILENAME Line 11, column 5: Role 'X' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 12, column 5: Role 'X' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 13, column 5: Role 'X' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 14, column 5: Role 'X' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 15, column 5: Role 'X' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 16, column 5: Role 'X' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 12, column 5: Role 'X' is already declared for type 'B2'
+$FILENAME Line 13, column 5: Role 'X' is already declared for type 'B2'
+$FILENAME Line 14, column 5: Role 'X' is already declared for type 'B2'
+$FILENAME Line 15, column 5: Role 'X' is already declared for type 'B2'
+$FILENAME Line 16, column 5: Role 'X' is already declared for type 'B2'
+
+$FILENAME Line 18, column 5: Role 'MyName' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 19, column 5: Role 'MyName' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 20, column 5: Role 'MyName' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 21, column 5: Role 'MyName' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 22, column 5: Role 'MyName' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 23, column 5: Role 'MyName' is an invalid redefinition for type 'B2', conflicts with supertype 'A'
+$FILENAME Line 19, column 5: Role 'MyName' is already declared for type 'B2'
+$FILENAME Line 20, column 5: Role 'MyName' is already declared for type 'B2'
+$FILENAME Line 21, column 5: Role 'MyName' is already declared for type 'B2'
+$FILENAME Line 22, column 5: Role 'MyName' is already declared for type 'B2'
+$FILENAME Line 23, column 5: Role 'MyName' is already declared for type 'B2'
+
+$FILENAME Line 28, column 12: Token 'X' is an invalid redefinition for type 'D3', conflicts with supertype 'C'
+$FILENAME Line 30, column 5: Role 'X' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 31, column 5: Role 'X' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 32, column 5: Role 'X' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 33, column 5: Role 'X' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 34, column 5: Role 'X' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 35, column 5: Role 'X' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 31, column 5: Role 'X' is already declared for type 'D2'
+$FILENAME Line 32, column 5: Role 'X' is already declared for type 'D2'
+$FILENAME Line 33, column 5: Role 'X' is already declared for type 'D2'
+$FILENAME Line 34, column 5: Role 'X' is already declared for type 'D2'
+$FILENAME Line 35, column 5: Role 'X' is already declared for type 'D2'
+
+$FILENAME Line 37, column 5: Role 'MyName' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 38, column 5: Role 'MyName' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 39, column 5: Role 'MyName' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 40, column 5: Role 'MyName' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 41, column 5: Role 'MyName' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 42, column 5: Role 'MyName' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 38, column 5: Role 'MyName' is already declared for type 'D2'
+$FILENAME Line 39, column 5: Role 'MyName' is already declared for type 'D2'
+$FILENAME Line 40, column 5: Role 'MyName' is already declared for type 'D2'
+$FILENAME Line 41, column 5: Role 'MyName' is already declared for type 'D2'
+$FILENAME Line 42, column 5: Role 'MyName' is already declared for type 'D2'
+
+$FILENAME Line 44, column 5: Role 'X2' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 45, column 5: Role 'X2' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 46, column 5: Role 'X2' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 47, column 5: Role 'X2' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 48, column 5: Role 'X2' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 49, column 5: Role 'X2' is an invalid redefinition for type 'D2', conflicts with supertype 'C'
+$FILENAME Line 45, column 5: Role 'X2' is already declared for type 'D2'
+$FILENAME Line 46, column 5: Role 'X2' is already declared for type 'D2'
+$FILENAME Line 47, column 5: Role 'X2' is already declared for type 'D2'
+$FILENAME Line 48, column 5: Role 'X2' is already declared for type 'D2'
+$FILENAME Line 49, column 5: Role 'X2' is already declared for type 'D2'

src/test/jastadd/errors/Inheritance.relast

-A ::= X;
-B1 : A ::= X;
+A ::= X MyName:X;
+B1 : A ::= X MyName:X MyName:Y;
 B2 : A;
-B3 : A ::= <X:String>;
+B3 : A ::= <X:String> <MyName:String>;
+// B4 has a child named X of type SubX
+B4 : A ::= X:SubX;
 X;
+SubX : X;
+Y;
 
 rel B2.X -> X;
+rel B2.X -> Y;
+rel B2.X? -> X;
+rel B2.X? -> Y;
+rel B2.X* -> X;
+rel B2.X* -> Y;
+
+rel B2.MyName -> X;
+rel B2.MyName -> Y;
+rel B2.MyName? -> X;
+rel B2.MyName? -> Y;
+rel B2.MyName* -> X;
+rel B2.MyName* -> Y;
+
+C ::= /X/ /MyName:X/ /<X2:long>/ ;
+D1 : C ::= X;
+D2 : C;
+D3 : C ::= <X:String> ;
+
+rel D2.X -> X;
+rel D2.X -> Y;
+rel D2.X? -> X;
+rel D2.X? -> Y;
+rel D2.X* -> X;
+rel D2.X* -> Y;
+
+rel D2.MyName -> X;
+rel D2.MyName -> Y;
+rel D2.MyName? -> X;
+rel D2.MyName? -> Y;
+rel D2.MyName* -> X;
+rel D2.MyName* -> Y;
+
+rel D2.X2 -> X;
+rel D2.X2 -> Y;
+rel D2.X2? -> X;
+rel D2.X2? -> Y;
+rel D2.X2* -> X;
+rel D2.X2* -> Y;

src/test/java/org/jastadd/relast/tests/Errors.java

@@ -10,6 +10,7 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 
@@ -23,32 +24,32 @@ class Errors {
   private static final String FILENAME_PATTERN = "$FILENAME";
 
   @Test
-  void test1() throws IOException {
+  void testStandardErrors() throws IOException {
     test("Errors");
   }
 
   @Test
-  void test1Left() throws IOException {
+  void testStandardErrorsLeft() throws IOException {
     test("ErrorsLeft");
   }
 
   @Test
-  void test2() throws IOException {
+  void testInheritance() throws IOException {
     test("Inheritance");
   }
 
   @Test
-  void test2Left() throws IOException {
+  void testInheritanceLeft() throws IOException {
     test("InheritanceLeft");
   }
 
   @Test
-  void test3() throws IOException {
+  void testMultipleFiles() throws IOException {
     test("Multiple", "Multiple_1", "Multiple_2");
   }
 
   @Test
-  void test3Left() throws IOException {
+  void testMultipleFilesLeft() throws IOException {
     test("MultipleLeft", "MultipleLeft_1", "MultipleLeft_2");
   }
 
@@ -60,7 +61,7 @@ class Errors {
     String expectedFile = "./src/test/jastadd/errors/" + name + ".expected";
 
     try {
-      System.out.println(System.getProperty("user.dir"));
+      logger.debug("user.dir: {}", System.getProperty("user.dir"));
       int returnValue = exec(Compiler.class, inFiles.toArray(new String[0]), new File(outFile));
       Assertions.assertEquals(1, returnValue, "Relast did not return with value 1");
     } catch (IOException | InterruptedException e) {
@@ -76,7 +77,14 @@ class Errors {
         expected = expected.replace(FILENAME_PATTERN + (i + 1), inFiles.get(i));
       }
     }
-    Assertions.assertEquals(expected, out);
+    List<String> outList = Arrays.asList(out.split("\n"));
+    Collections.sort(outList);
+    List<String> expectedList = Arrays.stream(expected.split("\n"))
+        .sorted()
+        .filter(s -> !s.isEmpty() && !s.startsWith("//"))
+        .collect(Collectors.toList());
+
+    Assertions.assertLinesMatch(expectedList, outList);
 
     logger.info("relast for " + name + " returned \n{}", out);
   }