Commit 4c57f378 authored by Johannes Mey's avatar Johannes Mey
Browse files

separate relast preprocessor from ros2rag

parent f9dc7f4a
Pipeline #6490 passed with stage
in 1 minute and 43 seconds
build
src/gen-res/
src/gen/
out/
*.class
apply plugin: 'java'
apply plugin: 'jastadd'
apply plugin: 'application'
apply plugin: "idea"
sourceCompatibility = 1.8
mainClassName = 'org.jastadd.relast.compiler.Compiler'
repositories {
jcenter()
}
buildscript {
repositories.jcenter()
dependencies {
classpath 'org.jastadd:jastaddgradle:1.13.3'
}
}
dependencies {
testImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.0'
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.4.0'
testCompile 'org.assertj:assertj-core:3.12.1'
compile 'com.fasterxml.jackson.core:jackson-core:2.9.8'
compile 'com.fasterxml.jackson.core:jackson-databind:2.9.8'
compile 'org.jastadd:jastadd:2.3.4'
runtime 'org.jastadd:jastadd:2.3.4'
compile group: 'net.sf.beaver', name: 'beaver-rt', version: '0.9.11'
}
sourceSets {
main {
java.srcDir "src/gen/java"
java.srcDir "buildSrc/gen/java"
}
}
test {
useJUnitPlatform()
maxHeapSize = '1G'
}
jar {
manifest {
attributes "Main-Class": 'org.jastadd.relast.compiler.Compiler'
}
from {
configurations.compile.collect { it.isDirectory() ? it : zipTree(it) }
}
}
task relast(type: JavaExec) {
group = 'Build'
main = "-jar"
doFirst {
delete "src/gen/jastadd/*.ast"
delete "src/gen/jastadd/RelAst.jadd"
delete "src/gen/jastadd/RelAstRefResolver.jadd"
delete "src/gen/jastadd/RelAstResolverStubs.jrag"
mkdir "src/gen/jastadd/"
}
args = [
"../libs/relast.jar",
"./src/main/jastadd/RelAst.relast",
"--listClass=java.util.ArrayList",
"--jastAddList=JastAddList",
"--useJastAddNames",
"--file",
"--resolverHelper",
"--grammarName=./src/gen/jastadd/RelAst"
]
inputs.files file("../libs/relast.jar"),
file("src/main/jastadd/RelAst.relast")
outputs.files file("./src/gen/jastadd/RelAst.ast"),
file("src/gen/jastadd/RelAst.jadd"),
file("src/gen/jastadd/RelAstRefResolver.jadd"),
file('src/gen/jastadd/RelAstResolverStubs.jrag')
}
jastadd {
configureModuleBuild()
modules {
//noinspection GroovyAssignabilityCheck
module("RelAst") {
java {
basedir "."
include "src/main/**/*.java"
include "src/gen/**/*.java"
}
jastadd {
basedir "."
include "src/main/jastadd/**/*.ast"
include "src/main/jastadd/**/*.jadd"
include "src/main/jastadd/**/*.jrag"
include "src/gen/jastadd/**/*.ast"
include "src/gen/jastadd/**/*.jadd"
include "src/gen/jastadd/**/*.jrag"
}
scanner {
include "src/main/jastadd/RelAst.flex"
}
parser {
include "src/main/jastadd/Preamble.parser"
include "src/main/jastadd/RelAst.parser"
}
}
}
cleanGen.doFirst {
delete "src/gen/java/org"
delete "src/gen-res/BuildInfo.properties"
}
preprocessParser.doFirst {
args += ["--no-beaver-symbol"]
}
module = "RelAst"
astPackage = 'org.jastadd.relast.ast'
parser.name = 'RelAstParser'
genDir = 'src/gen/java'
buildInfoDir = 'src/gen-res'
scanner.genDir = "src/gen/java/org/jastadd/relast/scanner"
parser.genDir = "src/gen/java/org/jastadd/relast/parser"
jastaddOptions = ["--lineColumnNumbers", "--List=JastAddList", "--safeLazy", "--visitCheck=true", "--rewrite=cnta", "--cache=all"]
}
generateAst.dependsOn relast
aspect NameResolution {
refine RefResolverStubs eq ASTNode.globallyResolveTypeDeclByToken(String id) = program().resolveTypeDecl(id);
syn TypeDecl Program.resolveTypeDecl(String name) {
for (TypeDecl decl : typeDecls()) {
if (decl.getName().equals(name)) {
return decl;
}
}
throw new RuntimeException("TypeDecl " + name + " could not be resolved.");
}
refine RefResolverStubs eq ASTNode.globallyResolveTokenComponentByToken(String id) {
// return a TokenComponent. id is of the form 'type_name + "." + token_name'
int dotIndex = id.indexOf(".");
String typeName = id.substring(0, dotIndex);
String tokenName = id.substring(dotIndex + 1);
TypeDecl type = program().resolveTypeDecl(typeName);
// iterate over components and find the matching tokenComponent
for (Component comp : type.getComponentList()) {
if (comp.isTokenComponent() && comp.getName().equals(tokenName)) {
return comp.asTokenComponent();
}
}
System.err.println("Could not resolve TokenComponent '" + id + "'.");
return null;
}
}
aspect Navigation {
// --- program ---
inh Program ASTNode.program();
eq Program.getChild().program() = this;
// --- typeDecls ---
coll java.util.Set<TypeDecl> Program.typeDecls() [new java.util.HashSet<>()] root Program;
TypeDecl contributes this
to Program.typeDecls()
for program();
// --- relations ---
coll java.util.Set<Relation> Program.relations() [new java.util.HashSet<>()] root Program;
Relation contributes this
to Program.relations()
for program();
// --- containingTypeDecl ---
inh TypeDecl Component.containingTypeDecl();
eq TypeDecl.getChild().containingTypeDecl() = this;
// --- containedFile ---
inh GrammarFile ASTNode.containedFile();
eq GrammarFile.getChild().containedFile() = this;
// --- isTokenComponent ---
syn boolean Component.isTokenComponent() = false;
eq TokenComponent.isTokenComponent() = true;
// --- asTokenComponent ---
syn TokenComponent Component.asTokenComponent() = null;
eq TokenComponent.asTokenComponent() = this;
}
%header {:
package org.jastadd.relast.parser;
import org.jastadd.relast.ast.*;
:};
%goal goal;
package org.jastadd.relast.scanner;
import org.jastadd.relast.parser.RelAstParser.Terminals;
%%
%public
%final
%class RelAstScanner
%extends beaver.Scanner
%type beaver.Symbol
%function nextToken
%yylexthrow beaver.Scanner.Exception
%scanerror RelAstScanner.ScannerError
%line
%column
%{
private StringBuilder stringLitSb = new StringBuilder();
private beaver.Symbol sym(short id) {
return new beaver.Symbol(id, yyline + 1, yycolumn + 1, yylength(), yytext());
}
private beaver.Symbol sym(short id, String text) {
return new beaver.Symbol(id, yyline + 1, yycolumn + 1, yylength(), text);
}
public static class ScannerError extends Error {
public ScannerError(String message) {
super(message);
}
}
%}
WhiteSpace = [ ] | \t | \f | \n | \r | \r\n
TraditionalComment = [/][*][^*]*[*]+([^*/][^*]*[*]+)*[/]
EndOfLineComment = "//" [^\n\r]*
Comment = {TraditionalComment} | {EndOfLineComment}
ID = [a-zA-Z$_][a-zA-Z0-9$_]*
%%
{WhiteSpace} { /* ignore */ }
{Comment} { return sym(Terminals.COMMENT); }
"abstract" { return sym(Terminals.ABSTRACT); }
"rel" { return sym(Terminals.RELATION); }
";" { return sym(Terminals.SCOL); }
":" { return sym(Terminals.COL); }
"::=" { return sym(Terminals.ASSIGN); }
"*" { return sym(Terminals.STAR); }
"." { return sym(Terminals.DOT); }
"," { return sym(Terminals.COMMA); }
"<" { return sym(Terminals.LT); }
">" { return sym(Terminals.GT); }
"[" { return sym(Terminals.LBRACKET); }
"]" { return sym(Terminals.RBRACKET); }
"/" { return sym(Terminals.SLASH); }
"?" { return sym(Terminals.QUESTION_MARK); }
"->" { return sym(Terminals.RIGHT); }
"<-" { return sym(Terminals.LEFT); }
"<->" { return sym(Terminals.BIDIRECTIONAL); }
// ID
{ID} { return sym(Terminals.ID); }
<<EOF>> { return sym(Terminals.EOF); }
[^] { throw new ScannerError((yyline+1) +"," + (yycolumn+1) + ": Illegal character <"+yytext()+">"); }
......@@ -17,14 +17,13 @@ aspect BackendAbstractGrammar {
}
}
public void GrammarFile.generateAbstractGrammar(StringBuilder b) {
b.append("// Grammar for file ").append(getFileName()).append("\n");
super.generateAbstractGrammar(b);
b.append("\n");
public String GrammarFile.generateAbstractGrammar() {
StringBuilder sb = new StringBuilder();
generateAbstractGrammar(sb);
return sb.toString();
}
public void Grammar.generateAbstractGrammar(StringBuilder b) {
for (Declaration decl : getDeclarationList()) {
decl.generateAbstractGrammar(b);
}
......
package org.jastadd.relast.compiler;
import beaver.Parser;
import org.jastadd.relast.ast.GrammarFile;
import org.jastadd.relast.ast.Program;
import org.jastadd.relast.compiler.options.CommandLine;
import org.jastadd.relast.compiler.options.CommandLine.CommandLineException;
import org.jastadd.relast.compiler.options.Option;
import org.jastadd.relast.compiler.options.StringOption;
import org.jastadd.relast.parser.RelAstParser;
import org.jastadd.relast.scanner.RelAstScanner;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.PrintWriter;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
public class Compiler {
private StringOption optionOutputDir;
private StringOption optionInputDir;
private ArrayList<Option<?>> options;
private CommandLine commandLine;
public Compiler() {
options = new ArrayList<>();
addOptions();
}
public static void main(String[] args) {
try {
new Compiler().run(args);
} catch (CommandLineException | CompilerException e) {
System.err.println(e.getMessage());
System.exit(-1);
}
}
public int run(String[] args) throws CommandLineException, CompilerException {
options = new ArrayList<>();
addOptions();
commandLine = new CommandLine(options);
commandLine.parse(args);
Path inputPath;
if (optionInputDir.isSet()) {
inputPath = Paths.get(optionInputDir.getValue());
} else {
inputPath = Paths.get(".");
printMessage("No input dir is set. Assuming current directory '" + inputPath.toAbsolutePath().toString() + "'.");
}
if (!inputPath.toFile().exists()) {
printMessage("Input path '" + inputPath.toAbsolutePath().toString() + "' does not exist. Exiting...");
System.exit(-1);
} else if (!inputPath.toFile().isDirectory()) {
printMessage("Input path '" + inputPath.toAbsolutePath().toString() + "' is not a directory. Exiting...");
System.exit(-1);
}
Path outputPath; // should not be used, but otherwise there is a compiler warning
if (optionOutputDir.isSet()) {
outputPath = Paths.get(optionOutputDir.getValue());
} else {
outputPath = Paths.get("./gen/");
printMessage("No output dir is set. Assuming '" + outputPath.toAbsolutePath().toString() + "'.");
}
if (outputPath.toFile().exists() && !outputPath.toFile().isDirectory()) {
printMessage("Output path '" + inputPath.toAbsolutePath().toString() + "' exists, but is not a directory. Exiting...");
}
printMessage("Running RelAST Preprocessor");
List<String> otherArgs = commandLine.getArguments();
if (!otherArgs.isEmpty()) {
printMessage("Unsupported arguments will be ignored: " + otherArgs);
}
Program program = parseProgram(inputPath);
printMessage("Writing output files");
for (GrammarFile grammarFile : program.getGrammarFileList()) {
// TODO decide and document what the file name should be, the full path or a simple name?
writeToFile(outputPath + grammarFile.getFileName() + "/Grammar.relast", grammarFile.generateAbstractGrammar());
}
writeToFile(outputPath + "/Grammar.relast", program.generateAbstractGrammar());
return 0;
}
private void printMessage(String message) {
System.out.println(message);
}
private void writeToFile(String filename, String str) throws CompilerException {
try {
PrintWriter writer = new PrintWriter(filename);
writer.print(str);
writer.close();
} catch (Exception e) {
throw new CompilerException("Could not write to file " + filename, e);
}
}
private void addOptions() {
optionOutputDir = addOption(new StringOption("outputDir", "target directory for the generated files."));
optionInputDir = addOption(new StringOption("inputDir", "input directory."));
}
private <OptionType extends Option<?>> OptionType addOption(OptionType option) {
options.add(option);
return option;
}
private Program parseProgram(Path inputPath) throws CompilerException {
Program program = new Program();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(inputPath, "*.relast")) {
RelAstParser parser = new RelAstParser();
stream.forEach(path -> {
try (BufferedReader reader = Files.newBufferedReader(path)) {
RelAstScanner scanner = new RelAstScanner(reader);
GrammarFile inputGrammar = (GrammarFile) parser.parse(scanner);
inputGrammar.dumpTree(System.out);
program.addGrammarFile(inputGrammar);
inputGrammar.treeResolveAll();
} catch (IOException | Parser.Exception e) {
printMessage("Could not parse grammar file " + path);
e.printStackTrace();
}
});
} catch (IOException e) {
printMessage("Unable to iterate over input path '" + inputPath.toAbsolutePath().toString() + "'. Exiting...");
e.printStackTrace();
System.exit(-1);
}
return program;
}
protected int error(String message) {
System.err.println("Error: " + message);
System.err.println();
System.err.println("Usage: java -jar relast.jar [--option1] [--option2=value] ... <filename1> <filename2> ... ");
System.err.println("Options:");
System.err.print(commandLine.printOptionHelp());
return 1;
}
public static class CompilerException extends Exception {
CompilerException(String message, Throwable cause) {
super(message, cause);
}
}
}
package org.jastadd.relast.compiler;
import java.util.*;
import java.util.function.Predicate;
import static java.util.stream.Collectors.toList;
public class Utils {
public static <T> List<T> filterToList(Collection<T> collection, Predicate<T> predicate) {
return collection.stream().filter(predicate).collect(toList());
}
public static <T> Set<T> asSet(T... t) {
return new HashSet<T>(Arrays.asList(t));
}
}
package org.jastadd.relast.compiler.options;
import java.util.*;
public class CommandLine {
private final Collection<Option<?>> options;
private final Map<String, Option<?>> mapping;
private final List<String> arguments;
public CommandLine(Collection<Option<?>> options) {
this.options = options;
this.mapping = new HashMap<>();
for (Option<?> option : options) {
mapping.put(option.getName(), option);
}
this.arguments = new ArrayList<>();
}
public void parse(String[] args) throws CommandLineException {
int i = 0;
while (i < args.length) {
if (args[i].startsWith(Option.PREFIX)) {
int argumentIndex = args[i].indexOf("=");
String name;
String argument = null;
if (argumentIndex > 0) {
name = args[i].substring(2, argumentIndex);
argument = args[i].substring(argumentIndex + 1);
} else {
name = args[i].substring(2);
}
Option<?> option = mapping.get(name);
if (option == null) {
throw new CommandLineException("Option " + Option.PREFIX + name + " not found");
}
match(option, argument);
} else {
arguments.add(args[i]);
}
i++;
}
}
public void match(Option<?> option, String argument) throws CommandLineException {
try {
switch (option.hasArgument()) {
case NO:
if (argument == null) {
option.match(null);
} else {
throw new CommandLineException("Option " + option + " is not allowed to have an argument");
}
break;
case OPTIONAL:
option.match(argument);
break;
case YES:
if (argument != null) {
option.match(argument);
} else {
throw new CommandLineException("Option " + option + " requires an argument");
}
break;
}
} catch (Option.IllegalMatchException e) {
throw new CommandLineException("Invalid value for option " + option + ": " + e.getMessage());
}
}
public List<String> getArguments() {
return arguments;