Skip to content
Snippets Groups Projects
Commit 4c57f378 authored by Johannes Mey's avatar Johannes Mey
Browse files

separate relast preprocessor from ros2rag

parent f9dc7f4a
No related branches found
No related tags found
No related merge requests found
Pipeline #6490 passed
Showing
with 822 additions and 5 deletions
build
src/gen-res/
src/gen/
out/
*.class
apply plugin: 'java'
apply plugin: 'jastadd'
apply plugin: 'application'
apply plugin: "idea"
sourceCompatibility = 1.8
mainClassName = 'org.jastadd.relast.compiler.Compiler'
repositories {
jcenter()
}
buildscript {
repositories.jcenter()
dependencies {
classpath 'org.jastadd:jastaddgradle:1.13.3'
}
}
dependencies {
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.0'
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.4.0'
testCompile 'org.assertj:assertj-core:3.12.1'
compile 'com.fasterxml.jackson.core:jackson-core:2.9.8'
compile 'com.fasterxml.jackson.core:jackson-databind:2.9.8'
compile 'org.jastadd:jastadd:2.3.4'
runtime 'org.jastadd:jastadd:2.3.4'
compile group: 'net.sf.beaver', name: 'beaver-rt', version: '0.9.11'
}
sourceSets {
main {
java.srcDir "src/gen/java"
java.srcDir "buildSrc/gen/java"
}
}
test {
useJUnitPlatform()
maxHeapSize = '1G'
}
jar {
manifest {
attributes "Main-Class": 'org.jastadd.relast.compiler.Compiler'
}
from {
configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }
}
}
task relast(type: JavaExec) {
group = 'Build'
main = "-jar"
doFirst {
delete "src/gen/jastadd/*.ast"
delete "src/gen/jastadd/RelAst.jadd"
delete "src/gen/jastadd/RelAstRefResolver.jadd"
delete "src/gen/jastadd/RelAstResolverStubs.jrag"
mkdir "src/gen/jastadd/"
}
args = [
"../libs/relast.jar",
"./src/main/jastadd/RelAst.relast",
"--listClass=java.util.ArrayList",
"--jastAddList=JastAddList",
"--useJastAddNames",
"--file",
"--resolverHelper",
"--grammarName=./src/gen/jastadd/RelAst"
]
inputs.files file("../libs/relast.jar"),
file("src/main/jastadd/RelAst.relast")
outputs.files file("./src/gen/jastadd/RelAst.ast"),
file("src/gen/jastadd/RelAst.jadd"),
file("src/gen/jastadd/RelAstRefResolver.jadd"),
file('src/gen/jastadd/RelAstResolverStubs.jrag')
}
jastadd {
configureModuleBuild()
modules {
//noinspection GroovyAssignabilityCheck
module("RelAst") {
java {
basedir "."
include "src/main/**/*.java"
include "src/gen/**/*.java"
}
jastadd {
basedir "."
include "src/main/jastadd/**/*.ast"
include "src/main/jastadd/**/*.jadd"
include "src/main/jastadd/**/*.jrag"
include "src/gen/jastadd/**/*.ast"
include "src/gen/jastadd/**/*.jadd"
include "src/gen/jastadd/**/*.jrag"
}
scanner {
include "src/main/jastadd/RelAst.flex"
}
parser {
include "src/main/jastadd/Preamble.parser"
include "src/main/jastadd/RelAst.parser"
}
}
}
cleanGen.doFirst {
delete "src/gen/java/org"
delete "src/gen-res/BuildInfo.properties"
}
preprocessParser.doFirst {
args += ["--no-beaver-symbol"]
}
module = "RelAst"
astPackage = 'org.jastadd.relast.ast'
parser.name = 'RelAstParser'
genDir = 'src/gen/java'
buildInfoDir = 'src/gen-res'
scanner.genDir = "src/gen/java/org/jastadd/relast/scanner"
parser.genDir = "src/gen/java/org/jastadd/relast/parser"
jastaddOptions = ["--lineColumnNumbers", "--List=JastAddList", "--safeLazy", "--visitCheck=true", "--rewrite=cnta", "--cache=all"]
}
generateAst.dependsOn relast
aspect Analysis {
}
aspect NameResolution {
refine RefResolverStubs eq ASTNode.globallyResolveTypeDeclByToken(String id) = program().resolveTypeDecl(id);
syn TypeDecl Program.resolveTypeDecl(String name) {
for (TypeDecl decl : typeDecls()) {
if (decl.getName().equals(name)) {
return decl;
}
}
throw new RuntimeException("TypeDecl " + name + " could not be resolved.");
}
refine RefResolverStubs eq ASTNode.globallyResolveTokenComponentByToken(String id) {
// return a TokenComponent. id is of the form 'type_name + "." + token_name'
int dotIndex = id.indexOf(".");
String typeName = id.substring(0, dotIndex);
String tokenName = id.substring(dotIndex + 1);
TypeDecl type = program().resolveTypeDecl(typeName);
// iterate over components and find the matching tokenComponent
for (Component comp : type.getComponentList()) {
if (comp.isTokenComponent() && comp.getName().equals(tokenName)) {
return comp.asTokenComponent();
}
}
System.err.println("Could not resolve TokenComponent '" + id + "'.");
return null;
}
}
aspect Navigation {
// --- program ---
inh Program ASTNode.program();
eq Program.getChild().program() = this;
// --- typeDecls ---
coll java.util.Set<TypeDecl> Program.typeDecls() [new java.util.HashSet<>()] root Program;
TypeDecl contributes this
to Program.typeDecls()
for program();
// --- relations ---
coll java.util.Set<Relation> Program.relations() [new java.util.HashSet<>()] root Program;
Relation contributes this
to Program.relations()
for program();
// --- containingTypeDecl ---
inh TypeDecl Component.containingTypeDecl();
eq TypeDecl.getChild().containingTypeDecl() = this;
// --- containedFile ---
inh GrammarFile ASTNode.containedFile();
eq GrammarFile.getChild().containedFile() = this;
// --- isTokenComponent ---
syn boolean Component.isTokenComponent() = false;
eq TokenComponent.isTokenComponent() = true;
// --- asTokenComponent ---
syn TokenComponent Component.asTokenComponent() = null;
eq TokenComponent.asTokenComponent() = this;
}
%header {:
package org.jastadd.relast.parser;
import org.jastadd.relast.ast.*;
:};
%goal goal;
package org.jastadd.relast.scanner;
import org.jastadd.relast.parser.RelAstParser.Terminals;
%%
%public
%final
%class RelAstScanner
%extends beaver.Scanner
%type beaver.Symbol
%function nextToken
%yylexthrow beaver.Scanner.Exception
%scanerror RelAstScanner.ScannerError
%line
%column
%{
private StringBuilder stringLitSb = new StringBuilder();
private beaver.Symbol sym(short id) {
return new beaver.Symbol(id, yyline + 1, yycolumn + 1, yylength(), yytext());
}
private beaver.Symbol sym(short id, String text) {
return new beaver.Symbol(id, yyline + 1, yycolumn + 1, yylength(), text);
}
public static class ScannerError extends Error {
public ScannerError(String message) {
super(message);
}
}
%}
WhiteSpace = [ ] | \t | \f | \n | \r | \r\n
TraditionalComment = [/][*][^*]*[*]+([^*/][^*]*[*]+)*[/]
EndOfLineComment = "//" [^\n\r]*
Comment = {TraditionalComment} | {EndOfLineComment}
ID = [a-zA-Z$_][a-zA-Z0-9$_]*
%%
{WhiteSpace} { /* ignore */ }
{Comment} { return sym(Terminals.COMMENT); }
"abstract" { return sym(Terminals.ABSTRACT); }
"rel" { return sym(Terminals.RELATION); }
";" { return sym(Terminals.SCOL); }
":" { return sym(Terminals.COL); }
"::=" { return sym(Terminals.ASSIGN); }
"*" { return sym(Terminals.STAR); }
"." { return sym(Terminals.DOT); }
"," { return sym(Terminals.COMMA); }
"<" { return sym(Terminals.LT); }
">" { return sym(Terminals.GT); }
"[" { return sym(Terminals.LBRACKET); }
"]" { return sym(Terminals.RBRACKET); }
"/" { return sym(Terminals.SLASH); }
"?" { return sym(Terminals.QUESTION_MARK); }
"->" { return sym(Terminals.RIGHT); }
"<-" { return sym(Terminals.LEFT); }
"<->" { return sym(Terminals.BIDIRECTIONAL); }
// ID
{ID} { return sym(Terminals.ID); }
<<EOF>> { return sym(Terminals.EOF); }
[^] { throw new ScannerError((yyline+1) +"," + (yycolumn+1) + ": Illegal character <"+yytext()+">"); }
......@@ -17,14 +17,13 @@ aspect BackendAbstractGrammar {
}
}
public void GrammarFile.generateAbstractGrammar(StringBuilder b) {
b.append("// Grammar for file ").append(getFileName()).append("\n");
super.generateAbstractGrammar(b);
b.append("\n");
public String GrammarFile.generateAbstractGrammar() {
StringBuilder sb = new StringBuilder();
generateAbstractGrammar(sb);
return sb.toString();
}
public void Grammar.generateAbstractGrammar(StringBuilder b) {
for (Declaration decl : getDeclarationList()) {
decl.generateAbstractGrammar(b);
}
......
package org.jastadd.relast.compiler;
import beaver.Parser;
import org.jastadd.relast.ast.GrammarFile;
import org.jastadd.relast.ast.Program;
import org.jastadd.relast.compiler.options.CommandLine;
import org.jastadd.relast.compiler.options.CommandLine.CommandLineException;
import org.jastadd.relast.compiler.options.Option;
import org.jastadd.relast.compiler.options.StringOption;
import org.jastadd.relast.parser.RelAstParser;
import org.jastadd.relast.scanner.RelAstScanner;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
public class Compiler {
private StringOption optionOutputDir;
private StringOption optionInputDir;
private ArrayList<Option<?>> options;
private CommandLine commandLine;
public Compiler() {
options = new ArrayList<>();
addOptions();
}
public static void main(String[] args) {
try {
new Compiler().run(args);
} catch (CommandLineException | CompilerException e) {
System.err.println(e.getMessage());
System.exit(-1);
}
}
public int run(String[] args) throws CommandLineException, CompilerException {
options = new ArrayList<>();
addOptions();
commandLine = new CommandLine(options);
commandLine.parse(args);
Path inputPath;
if (optionInputDir.isSet()) {
inputPath = Paths.get(optionInputDir.getValue());
} else {
inputPath = Paths.get(".");
printMessage("No input dir is set. Assuming current directory '" + inputPath.toAbsolutePath().toString() + "'.");
}
if (!inputPath.toFile().exists()) {
printMessage("Input path '" + inputPath.toAbsolutePath().toString() + "' does not exist. Exiting...");
System.exit(-1);
} else if (!inputPath.toFile().isDirectory()) {
printMessage("Input path '" + inputPath.toAbsolutePath().toString() + "' is not a directory. Exiting...");
System.exit(-1);
}
Path outputPath; // should not be used, but otherwise there is a compiler warning
if (optionOutputDir.isSet()) {
outputPath = Paths.get(optionOutputDir.getValue());
} else {
outputPath = Paths.get("./gen/");
printMessage("No output dir is set. Assuming '" + outputPath.toAbsolutePath().toString() + "'.");
}
if (outputPath.toFile().exists() && !outputPath.toFile().isDirectory()) {
printMessage("Output path '" + inputPath.toAbsolutePath().toString() + "' exists, but is not a directory. Exiting...");
}
printMessage("Running RelAST Preprocessor");
List<String> otherArgs = commandLine.getArguments();
if (!otherArgs.isEmpty()) {
printMessage("Unsupported arguments will be ignored: " + otherArgs);
}
Program program = parseProgram(inputPath);
printMessage("Writing output files");
for (GrammarFile grammarFile : program.getGrammarFileList()) {
// TODO decide and document what the file name should be, the full path or a simple name?
writeToFile(outputPath + grammarFile.getFileName() + "/Grammar.relast", grammarFile.generateAbstractGrammar());
}
writeToFile(outputPath + "/Grammar.relast", program.generateAbstractGrammar());
return 0;
}
private void printMessage(String message) {
System.out.println(message);
}
private void writeToFile(String filename, String str) throws CompilerException {
try {
PrintWriter writer = new PrintWriter(filename);
writer.print(str);
writer.close();
} catch (Exception e) {
throw new CompilerException("Could not write to file " + filename, e);
}
}
private void addOptions() {
optionOutputDir = addOption(new StringOption("outputDir", "target directory for the generated files."));
optionInputDir = addOption(new StringOption("inputDir", "input directory."));
}
private <OptionType extends Option<?>> OptionType addOption(OptionType option) {
options.add(option);
return option;
}
private Program parseProgram(Path inputPath) throws CompilerException {
Program program = new Program();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(inputPath, "*.relast")) {
RelAstParser parser = new RelAstParser();
stream.forEach(path -> {
try (BufferedReader reader = Files.newBufferedReader(path)) {
RelAstScanner scanner = new RelAstScanner(reader);
GrammarFile inputGrammar = (GrammarFile) parser.parse(scanner);
inputGrammar.dumpTree(System.out);
program.addGrammarFile(inputGrammar);
inputGrammar.treeResolveAll();
} catch (IOException | Parser.Exception e) {
printMessage("Could not parse grammar file " + path);
e.printStackTrace();
}
});
} catch (IOException e) {
printMessage("Unable to iterate over input path '" + inputPath.toAbsolutePath().toString() + "'. Exiting...");
e.printStackTrace();
System.exit(-1);
}
return program;
}
protected int error(String message) {
System.err.println("Error: " + message);
System.err.println();
System.err.println("Usage: java -jar relast.jar [--option1] [--option2=value] ... <filename1> <filename2> ... ");
System.err.println("Options:");
System.err.print(commandLine.printOptionHelp());
return 1;
}
public static class CompilerException extends Exception {
CompilerException(String message, Throwable cause) {
super(message, cause);
}
}
}
package org.jastadd.relast.compiler;
import java.util.*;
import java.util.function.Predicate;
import static java.util.stream.Collectors.toList;
public class Utils {
public static <T> List<T> filterToList(Collection<T> collection, Predicate<T> predicate) {
return collection.stream().filter(predicate).collect(toList());
}
public static <T> Set<T> asSet(T... t) {
return new HashSet<T>(Arrays.asList(t));
}
}
package org.jastadd.relast.compiler.options;
import java.util.*;
public class CommandLine {
private final Collection<Option<?>> options;
private final Map<String, Option<?>> mapping;
private final List<String> arguments;
public CommandLine(Collection<Option<?>> options) {
this.options = options;
this.mapping = new HashMap<>();
for (Option<?> option : options) {
mapping.put(option.getName(), option);
}
this.arguments = new ArrayList<>();
}
public void parse(String[] args) throws CommandLineException {
int i = 0;
while (i < args.length) {
if (args[i].startsWith(Option.PREFIX)) {
int argumentIndex = args[i].indexOf("=");
String name;
String argument = null;
if (argumentIndex > 0) {
name = args[i].substring(2, argumentIndex);
argument = args[i].substring(argumentIndex + 1);
} else {
name = args[i].substring(2);
}
Option<?> option = mapping.get(name);
if (option == null) {
throw new CommandLineException("Option " + Option.PREFIX + name + " not found");
}
match(option, argument);
} else {
arguments.add(args[i]);
}
i++;
}
}
public void match(Option<?> option, String argument) throws CommandLineException {
try {
switch (option.hasArgument()) {
case NO:
if (argument == null) {
option.match(null);
} else {
throw new CommandLineException("Option " + option + " is not allowed to have an argument");
}
break;
case OPTIONAL:
option.match(argument);
break;
case YES:
if (argument != null) {
option.match(argument);
} else {
throw new CommandLineException("Option " + option + " requires an argument");
}
break;
}
} catch (Option.IllegalMatchException e) {
throw new CommandLineException("Invalid value for option " + option + ": " + e.getMessage());
}
}
public List<String> getArguments() {
return arguments;
}
public String printOptionHelp() {
StringBuilder sb = new StringBuilder();
int longestOption = 0;
for (Option<?> option : options) {
if (longestOption < option.getName().length()) {
longestOption = option.getName().length();
}
}
for (Option<?> option : new TreeSet<>(options)) {
String s = String.format(" %s%-" + (longestOption + 6) + "s %s%n",
Option.PREFIX, option.getName(), option.getDescription());
sb.append(s);
}
return sb.toString();
}
public static class CommandLineException extends Exception {
private static final long serialVersionUID = 1L;
public CommandLineException(String message) {
super(message);
}
}
}
package org.jastadd.relast.compiler.options;
import java.util.Collection;
import java.util.TreeSet;
public class EnumOption extends Option<String> {
private final TreeSet<String> allowedValues;
private final String defaultValue;
private String value;
private boolean isSet;
public EnumOption(String name, String description, Collection<String> allowedValues, String defaultValue) {
super(name, description);
this.allowedValues = new TreeSet<>(allowedValues);
this.defaultValue = defaultValue;
this.value = defaultValue;
this.isSet = false;
}
public boolean addAllowedValue(String allowedValue) {
return allowedValues.add(allowedValue);
}
@Override
public String getValue() {
return value;
}
@Override
public Option.HasArgument hasArgument() {
return Option.HasArgument.OPTIONAL;
}
@Override
public void match(String argument) throws IllegalMatchException {
if (argument == null) {
isSet = true;
value = defaultValue;
} else if (allowedValues.contains(argument)) {
isSet = true;
value = argument;
} else {
throw new IllegalMatchException(argument
+ " is not allowed, allowed values are " + allowedValues);
}
}
@Override
public boolean isSet() {
return isSet;
}
@Override
public String getDescription() {
String allowedValuesStr = allowedValues.toString();
allowedValuesStr = allowedValuesStr.substring(1);
allowedValuesStr = allowedValuesStr.substring(0, allowedValuesStr.length() - 1);
return super.getDescription() + " (allowed values: " + allowedValuesStr + ")";
}
}
package org.jastadd.relast.compiler.options;
public class FlagOption extends Option<Boolean> {
private boolean value;
public FlagOption(String name, String description) {
super(name, description);
value = false;
}
@Override
public Boolean getValue() {
return value;
}
@Override
public Option.HasArgument hasArgument() {
return Option.HasArgument.NO;
}
@Override
public void match(String string) throws IllegalMatchException {
value = true;
}
@Override
public boolean isSet() {
return getValue();
}
}
package org.jastadd.relast.compiler.options;
abstract public class Option<ValueType> implements Comparable<Option<?>> {
public final static String PREFIX = "--";
private final String name;
private final String description;
public Option(String name, String description) {
this.name = name;
this.description = description;
}
public String getName() {
return name;
}
public String getDescription() {
return description;
}
@Override
public int compareTo(Option<?> o) {
return name.compareTo(o.name);
}
@Override
public boolean equals(Object other) {
if (other instanceof Option) {
return compareTo((Option<?>) other) == 0;
}
return false;
}
@Override
public String toString() {
return PREFIX + name;
}
abstract public boolean isSet();
abstract public ValueType getValue();
abstract public HasArgument hasArgument();
abstract public void match(String input) throws IllegalMatchException;
public enum HasArgument {
NO,
OPTIONAL,
YES
}
public static class IllegalMatchException extends Exception {
private static final long serialVersionUID = 1L;
public IllegalMatchException(String message) {
super(message);
}
}
}
package org.jastadd.relast.compiler.options;
public class StringOption extends Option<String> {
private String value;
private boolean isSet;
public StringOption(String name, String description) {
this(name, description, "");
}
public StringOption(String name, String description, String defaultValue) {
super(name, description);
value = defaultValue;
isSet = false;
}
@Override
public String getValue() {
return value;
}
@Override
public Option.HasArgument hasArgument() {
return Option.HasArgument.YES;
}
@Override
public void match(String value) {
this.value = value;
isSet = true;
}
@Override
public boolean isSet() {
return isSet;
}
}
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="INFO">
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="info">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>
\ No newline at end of file
package org.jastadd.ros2rag.tests;
import org.jastadd.relast.compiler.Compiler;
import org.jastadd.relast.compiler.options.CommandLine;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.nio.file.Paths;
import java.util.Objects;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class RelAstTest {
void transform(String inputDir, String outputDir) throws CommandLine.CommandLineException, Compiler.CompilerException {
System.out.println("Running test in directory '" + Paths.get(".").toAbsolutePath() + "'.");
assertTrue(Paths.get(inputDir).toFile().exists(), "input directory does not exist");
assertTrue(Paths.get(inputDir).toFile().isDirectory(), "input directory is not a directory");
File outputDirFile = Paths.get(outputDir).toFile();
if (outputDirFile.exists()) {
assertTrue(outputDirFile.isDirectory());
if (Objects.requireNonNull(outputDirFile.list(), "Could not read output directory").length != 0) {
System.out.println("output directory is not empty!");
}
} else {
assertTrue(outputDirFile.mkdir());
}
String[] args = {
"--outputDir=" + outputDir,
"--inputDir=" + inputDir
};
new Compiler().run(args);
}
@Test
void transformMinimalExample() throws CommandLine.CommandLineException, Compiler.CompilerException {
transform("src/test/resources/in", "src/test/resources/out");
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment