Commit 570d1cbd authored by René Schöne's avatar René Schöne
Browse files

Update preprocessor and ragconnect-compiler.

- make compiler jastaddCompliant
- allow multiple input grammars and connect files
- compiler output are (possibly modified) input grammars, and always three files (MqttHandler.jadd, RagConnect.jadd and RagConnect.relast)
parent fc88a20e
Pipeline #7524 failed with stages
in 1 minute and 28 seconds
......@@ -17,12 +17,32 @@ buildscript {
}
dependencies {
implementation project(':relast.preprocessor')
implementation group: 'com.github.spullara.mustache.java', name: 'compiler', version: '0.9.6'
implementation group: 'org.apache.logging.log4j', name: 'log4j-jul', version: '2.11.2'
runtime group: 'org.jastadd', name: 'jastadd', version: '2.3.4'
api group: 'net.sf.beaver', name: 'beaver-rt', version: '0.9.11'
}
def versionFile = 'src/main/resources/ragConnectVersion.properties'
def oldProps = new Properties()
try {
file(versionFile).withInputStream { stream -> oldProps.load(stream) }
version = oldProps['version']
} catch (e) {
// this happens, if either the properties file is not present, or cannot be read from
throw new GradleException("File ${versionFile} not found or unreadable. Aborting.", e)
}
task newVersion() {
doFirst {
def props = new Properties()
props['version'] = value
props.store(file(versionFile).newWriter(), null)
}
}
sourceSets {
main {
java.srcDir "src/gen/java"
......
......@@ -9,31 +9,11 @@ aspect Navigation {
eq RagConnect.getChild().ragconnect() = this;
eq MRagConnect.getChild().ragconnect() = getRagConnect();
// --- containedFile (first equation should be in preprocessor) ---
eq Program.getChild().containedFile() = null;
// --- containedFile
eq RagConnect.getChild().containedFile() = null;
// --- isTypeComponent (should be in preprocessor) ---
syn boolean Component.isTypeComponent() = false;
eq TypeComponent.isTypeComponent() = true;
// --- asTypeComponent (should be in preprocessor) ---
syn TypeComponent Component.asTypeComponent() = null;
eq TypeComponent.asTypeComponent() = this;
// --- isListComponent (should be in preprocessor) ---
syn boolean Component.isListComponent() = false;
eq ListComponent.isListComponent() = true;
// --- asListComponent (should be in preprocessor) ---
syn ListComponent Component.asListComponent() = null;
eq ListComponent.asListComponent() = this;
// --- containedFileName (should replace containedFile in preprocessor) ---
inh String ASTNode.containedFileName();
eq GrammarFile.getChild().containedFileName() = getFileName();
// --- containedFileName ---
eq RagConnect.getChild().containedFileName() = getFileName();
eq Program.getChild().containedFileName() = null;
eq MRagConnect.getChild().containedFileName() = null;
// --- isTokenUpdateDefinition ---
......@@ -71,10 +51,4 @@ aspect Navigation {
// --- isDefaultMappingDefinition ---
syn boolean MappingDefinition.isDefaultMappingDefinition() = false;
eq DefaultMappingDefinition.isDefaultMappingDefinition() = true;
// --- allTokenComponents ---
coll java.util.Set<TokenComponent> Program.allTokenComponents() [new java.util.HashSet<>()] root Program;
TokenComponent contributes this
to Program.allTokenComponents()
for program();
}
......@@ -282,6 +282,7 @@ aspect RelationGeneration {
right.setType(getSource().containingTypeDecl());
result.setLeft(left);
result.setRight(right);
result.addComment(new WhitespaceComment("\n"));
return result;
}
}
......
......@@ -13,5 +13,8 @@ import org.jastadd.ragconnect.parser.RagConnectParser.Terminals;
%yylexthrow beaver.Scanner.Exception
%scanerror RagConnectScanner.ScannerError
%x COMMENT
%s DECLARATION
%line
%column
"read" { return sym(Terminals.READ); }
"write" { return sym(Terminals.WRITE); }
"read" { yybegin(DECLARATION); return sym(Terminals.READ); }
"write" { yybegin(DECLARATION); return sym(Terminals.WRITE); }
"using" { return sym(Terminals.USING); }
"canDependOn" { return sym(Terminals.CAN_DEPEND_ON); }
"maps" { return sym(Terminals.MAPS); }
......
package org.jastadd.ragconnect.compiler;
import beaver.Parser;
import org.jastadd.ragconnect.compiler.options.CommandLine;
import org.jastadd.ragconnect.compiler.options.FlagOption;
import org.jastadd.ragconnect.compiler.options.Option;
import org.jastadd.ragconnect.compiler.options.StringOption;
import org.jastadd.option.BooleanOption;
import org.jastadd.option.ValueOption;
import org.jastadd.relast.compiler.AbstractCompiler;
import org.jastadd.relast.compiler.CompilerException;
import org.jastadd.ragconnect.ast.*;
import org.jastadd.ragconnect.parser.RagConnectParser;
import org.jastadd.ragconnect.scanner.RagConnectScanner;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Collection;
import java.util.MissingResourceException;
import java.util.ResourceBundle;
public class Compiler {
public class Compiler extends AbstractCompiler {
private StringOption optionOutputDir;
private StringOption optionInputGrammar;
private StringOption optionRootNode;
private StringOption optionInputRagConnect;
private FlagOption optionHelp;
private FlagOption optionVerbose;
private FlagOption optionLogReads;
private FlagOption optionLogWrites;
private ArrayList<Option<?>> options;
private CommandLine commandLine;
// private ValueOption optionOutputDir;
private ValueOption optionRootNode;
private BooleanOption optionVerbose;
private BooleanOption optionLogReads;
private BooleanOption optionLogWrites;
public Compiler() {
options = new ArrayList<>();
addOptions();
}
public void run(String[] args) throws CommandLine.CommandLineException, CompilerException {
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
System.setProperty("mustache.debug", "true");
options = new ArrayList<>();
addOptions();
commandLine = new CommandLine(options);
commandLine.parse(args);
if (optionHelp.isSet()) {
printUsage();
return;
}
if (optionVerbose.isSet()) {
try {
run();
} catch (CompilerException e) {
e.printStackTrace();
throw e;
}
} else {
run();
}
super("ragconnect", true);
}
private void run() throws CompilerException {
String outputDir;
if (optionOutputDir.isSet()) {
outputDir = optionOutputDir.getValue();
} else {
outputDir = "gen";
System.out.println("No output output dir is set. Assuming '" + outputDir + "'.");
@Override
protected int compile() throws CompilerException {
if (getConfiguration().shouldPrintVersion()) {
System.out.println(readVersion());
return 0;
}
try {
Files.createDirectories(Paths.get(outputDir));
} catch (IOException e) {
throw new CompilerException("Error creating output dir " + outputDir, e);
if (getConfiguration().shouldPrintHelp()) {
getConfiguration().printHelp(System.out);
return 0;
}
printMessage("Running RagConnect Preprocessor");
printMessage("Running RagConnect " + readVersion());
if (anyRequiredOptionIsUnset()) {
throw new CompilerException("Aborting due to missing values for required options.");
if (!getConfiguration().outputDir().exists()) {
try {
Files.createDirectories(getConfiguration().outputDir().toPath());
} catch (IOException e) {
throw new CompilerException("Error creating output dir " + getConfiguration().outputDir(), e);
}
}
List<String> otherArgs = commandLine.getArguments();
if (!otherArgs.isEmpty()) {
printMessage("Superfluous arguments will be ignored: " + otherArgs);
}
RagConnect ragConnect = parseProgram(optionInputGrammar.getValue(), optionInputRagConnect.getValue());
RagConnect ragConnect = parseProgram(getConfiguration().getFiles());
if (!ragConnect.errors().isEmpty()) {
System.err.println("Errors:");
......@@ -103,114 +70,166 @@ public class Compiler {
throw new CompilerException("Could not open " + mqttHandlerFileName);
}
Files.copy(inputStream,
Paths.get(outputDir, mqttHandlerFileName),
getConfiguration().outputDir().toPath().resolve(mqttHandlerFileName),
StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
throw new CompilerException("Could not copy " + mqttHandlerFileName, e);
}
writeToFile(outputDir + "/Grammar.relast", ragConnect.getProgram().generateAbstractGrammar());
writeToFile(outputDir + "/RagConnect.jadd", ragConnect.generateAspect(optionRootNode.getValue()));
}
private boolean anyRequiredOptionIsUnset() {
boolean foundError = false;
for (Option<?> option : options) {
if (option.hasArgument() == Option.HasArgument.YES && !option.isSet()) {
System.err.println("Option '" + option.getName() +
"' (" + option.getDescription() + ") is required but unset!");
foundError = true;
}
for (GrammarFile grammarFile : ragConnect.getProgram().getGrammarFileList()) {
Path outputFile = getConfiguration().outputDir().toPath().resolve(grammarFile.getFileName());
writeToFile(outputFile, grammarFile.generateAbstractGrammar());
}
return foundError;
writeToFile(getConfiguration().outputDir().toPath().resolve("RagConnect.jadd"), ragConnect.generateAspect(optionRootNode.value()));
return 0;
}
public static void main(String[] args) {
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
System.setProperty("mustache.debug", "true");
try {
new Compiler().run(args);
} catch (CommandLine.CommandLineException | CompilerException e) {
} catch (CompilerException e) {
System.err.println(e.getMessage());
System.exit(-1);
}
}
/**
* Reads the version string.
*
* The version string is read from the property file
* src/main/resources/Version.properties. This
* file should be generated during the build process. If it is missing
* then there is some problem in the build script.
*
* @author Jesper Öqvist <jesper.oqvist@cs.lth.se>
* @return the read version string, or <code>version ?</code>
*/
private String readVersion() {
try {
ResourceBundle resources = ResourceBundle.getBundle("ragConnectVersion");
return resources.getString("version");
} catch (MissingResourceException e) {
return "version ?";
}
}
private void printMessage(String message) {
System.out.println(message);
}
private void writeToFile(String filename, String str) throws CompilerException {
try {
PrintWriter writer = new PrintWriter(filename);
writer.print(str);
writer.close();
private void writeToFile(Path path, String str) throws CompilerException {
try (BufferedWriter writer = Files.newBufferedWriter(path)) {
writer.append(str);
} catch (Exception e) {
throw new CompilerException("Could not write to file " + filename, e);
throw new CompilerException("Could not write to file " + path.toAbsolutePath(), e);
}
}
private void addOptions() {
optionOutputDir = addOption(new StringOption("outputDir", "target directory for the generated files."));
optionInputGrammar = addOption(new StringOption("inputGrammar", "base grammar."));
optionRootNode = addOption(new StringOption("rootNode", "root node in the base grammar."));
optionInputRagConnect = addOption(new StringOption("inputRagConnect", "connect definition file."));
optionHelp = addOption(new FlagOption("help", "Print usage and exit."));
optionVerbose = addOption(new FlagOption("verbose", "Print more messages while compiling."));
optionLogReads = addOption(new FlagOption("logReads", "Enable logging for every read."));
optionLogWrites = addOption(new FlagOption("logWrites", "Enable logging for every write."));
protected void initOptions() {
super.initOptions();
optionRootNode = addOption(
new ValueOption("rootNode", "root node in the base grammar.")
.acceptAnyValue()
.needsValue(true));
optionVerbose = addOption(
new BooleanOption("verbose", "Print more messages while compiling.")
.defaultValue(false));
optionLogReads = addOption(
new BooleanOption("logReads", "Enable logging for every read.")
.defaultValue(false));
optionLogWrites = addOption(
new BooleanOption("logWrites", "Enable logging for every write.")
.defaultValue(false));
}
private <OptionType extends Option<?>> OptionType addOption(OptionType option) {
options.add(option);
return option;
}
private RagConnect parseProgram(String inputGrammarFileName, String inputRagConnectFileName) throws CompilerException {
private RagConnect parseProgram(Collection<String> files) throws CompilerException {
Program program = new Program();
RagConnect ros2Rag;
GrammarFile inputGrammar;
try (BufferedReader reader = Files.newBufferedReader(Paths.get(inputGrammarFileName))) {
RagConnect ragConnect = new RagConnect();
ragConnect.setProgram(program);
GrammarFile ragConnectGrammarPart = new GrammarFile();
ragConnectGrammarPart.setFileName("RagConnect.relast");
program.addGrammarFile(ragConnectGrammarPart);
for (String filename : files) {
String extension = filename.substring(filename.lastIndexOf('.') + 1);
switch (extension) {
case "ast":
case "relast":
// processGrammar
parseGrammar(program, filename);
break;
case "connect":
// process ragConnect
RagConnect parsedRagConnect = parseRagConnect(program, filename);
mergeRagConnectDefinitions(ragConnect, parsedRagConnect);
break;
default:
throw new CompilerException("Unknown file extension in " + filename);
}
}
ragConnect.treeResolveAll();
ragConnect.additionalRelations().forEach(ragConnectGrammarPart::addDeclaration);
ASTNode.loggingEnabledForReads = optionLogReads.value();
ASTNode.loggingEnabledForWrites = optionLogWrites.value();
return ragConnect;
}
private void parseGrammar(Program program, String filename) throws CompilerException {
try (BufferedReader reader = Files.newBufferedReader(Paths.get(filename))) {
RagConnectScanner scanner = new RagConnectScanner(reader);
RagConnectParser parser = new RagConnectParser();
inputGrammar = (GrammarFile) parser.parse(scanner);
if (optionVerbose.isSet()) {
inputGrammar.dumpTree(System.out);
GrammarFile grammarFile = (GrammarFile) parser.parse(scanner);
if (optionVerbose.value()) {
grammarFile.dumpTree(System.out);
}
program.addGrammarFile(inputGrammar);
inputGrammar.treeResolveAll();
inputGrammar.setFileName(inputGrammarFileName);
program.addGrammarFile(grammarFile);
grammarFile.treeResolveAll();
grammarFile.setFileName(toBaseName(filename));
} catch (IOException | Parser.Exception e) {
throw new CompilerException("Could not parse grammar file " + inputGrammarFileName, e);
throw new CompilerException("Could not parse grammar file " + filename, e);
}
}
try (BufferedReader reader = Files.newBufferedReader(Paths.get(inputRagConnectFileName))) {
private RagConnect parseRagConnect(Program program, String filename) throws CompilerException {
try (BufferedReader reader = Files.newBufferedReader(Paths.get(filename))) {
RagConnectScanner scanner = new RagConnectScanner(reader);
RagConnectParser parser = new RagConnectParser();
ros2Rag = (RagConnect) parser.parse(scanner, RagConnectParser.AltGoals.ragconnect);
ros2Rag.setProgram(program);
ros2Rag.setFileName(inputRagConnectFileName);
RagConnect ragConnect = (RagConnect) parser.parse(scanner, RagConnectParser.AltGoals.ragconnect);
ragConnect.setProgram(program);
ragConnect.setFileName(toBaseName(filename));
return ragConnect;
} catch (IOException | Parser.Exception e) {
throw new CompilerException("Could not parse ros2rag file " + inputRagConnectFileName, e);
throw new CompilerException("Could not parse connect file " + filename, e);
}
ros2Rag.treeResolveAll();
ros2Rag.additionalRelations().forEach(inputGrammar::addDeclaration);
ASTNode.loggingEnabledForReads = optionLogReads.isSet();
ASTNode.loggingEnabledForWrites = optionLogWrites.isSet();
return ros2Rag;
}
protected void printUsage() {
System.out.println("Usage: java -jar ros2rag.jar [--option1] [--option2=value] ... <filename1> <filename2> ... ");
System.out.println("Options:");
System.out.print(commandLine.printOptionHelp());
/**
* Extracts the basename of the given file, with file extension
* @param filename the given filename
* @return the basename
*/
private String toBaseName(String filename) {
return new File(filename).getName();
}
public static class CompilerException extends Exception {
CompilerException(String message) {
super(message);
private void mergeRagConnectDefinitions(RagConnect ragConnect, RagConnect ragConnectToIntegrate) {
for (UpdateDefinition updateDefinition : ragConnectToIntegrate.getUpdateDefinitionList()) {
ragConnect.addUpdateDefinition(updateDefinition);
}
CompilerException(String message, Throwable cause) {
super(message, cause);
for (MappingDefinition mappingDefinition : ragConnectToIntegrate.getMappingDefinitionList()) {
ragConnect.addMappingDefinition(mappingDefinition);
}
for (DependencyDefinition dependencyDefinition : ragConnectToIntegrate.getDependencyDefinitionList()) {
ragConnect.addDependencyDefinition(dependencyDefinition);
}
}
// protected void printUsage() {
// System.out.println("Usage: java -jar ragconnect.jar [--option1] [--option2=value] ... <filename1> <filename2> ... ");
// System.out.println("Options:");
// System.out.print(commandLine.printOptionHelp());
// }
}
package org.jastadd.ragconnect.compiler.options;
import java.util.*;
public class CommandLine {
private final Collection<Option<?>> options;
private final Map<String, Option<?>> mapping;
private final List<String> arguments;
public CommandLine(Collection<Option<?>> options) {
this.options = options;
this.mapping = new HashMap<>();
for (Option<?> option : options) {
mapping.put(option.getName(), option);
}
this.arguments = new ArrayList<>();
}
public void parse(String[] args) throws CommandLineException {
int i = 0;
while (i < args.length) {
if (args[i].startsWith(Option.PREFIX)) {
int argumentIndex = args[i].indexOf("=");
String name;
String argument = null;
if (argumentIndex > 0) {
name = args[i].substring(2, argumentIndex);
argument = args[i].substring(argumentIndex + 1);
} else {
name = args[i].substring(2);
}
Option<?> option = mapping.get(name);
if (option == null) {
throw new CommandLineException("Option " + Option.PREFIX + name + " not found");
}
match(option, argument);
} else {
arguments.add(args[i]);
}
i++;
}
}
public void match(Option<?> option, String argument) throws CommandLineException {
try {
switch (option.hasArgument()) {
case NO:
if (argument == null) {
option.match(null);
} else {
throw new CommandLineException("Option " + option + " is not allowed to have an argument");
}
break;
case OPTIONAL:
option.match(argument);
break;
case YES:
if (argument != null) {
option.match(argument);
} else {
throw new CommandLineException("Option " + option + " requires an argument");
}
break;
}
} catch (Option.IllegalMatchException e) {
throw new CommandLineException("Invalid value for option " + option + ": " + e.getMessage());
}
}
public List<String> getArguments() {
return arguments;
}
public String printOptionHelp() {
StringBuilder sb = new StringBuilder();
int longestOption = 0;
for (Option<?> option : options) {
if (longestOption < option.getName().length()) {
longestOption = option.getName().length();
}
}
for (Option<?> option : new TreeSet<>(options)) {
String s = String.format(" %s%-" + (longestOption + 6) + "s %s%n",
Option.PREFIX, option.getName(), option.getDescription());
sb.append(s);
}
return sb.toString();
}
public static class CommandLineException extends Exception {
private static final long serialVersionUID = 1L;
public CommandLineException(String message) {
super(message);
}
}
}
package org.jastadd.ragconnect.compiler.options;
import java.util.Collection;
import java.util.TreeSet;