Commit 570d1cbd authored by René Schöne's avatar René Schöne
Browse files

Update preprocessor and ragconnect-compiler.

- make compiler jastaddCompliant
- allow multiple input grammars and connect files
- compiler output are (possibly modified) input grammars, and always three files (MqttHandler.jadd, RagConnect.jadd and RagConnect.relast)
parent fc88a20e
Pipeline #7524 failed with stages
in 1 minute and 28 seconds
...@@ -17,12 +17,32 @@ buildscript { ...@@ -17,12 +17,32 @@ buildscript {
} }
dependencies { dependencies {
implementation project(':relast.preprocessor')
implementation group: 'com.github.spullara.mustache.java', name: 'compiler', version: '0.9.6' implementation group: 'com.github.spullara.mustache.java', name: 'compiler', version: '0.9.6'
implementation group: 'org.apache.logging.log4j', name: 'log4j-jul', version: '2.11.2' implementation group: 'org.apache.logging.log4j', name: 'log4j-jul', version: '2.11.2'
runtime group: 'org.jastadd', name: 'jastadd', version: '2.3.4' runtime group: 'org.jastadd', name: 'jastadd', version: '2.3.4'
api group: 'net.sf.beaver', name: 'beaver-rt', version: '0.9.11' api group: 'net.sf.beaver', name: 'beaver-rt', version: '0.9.11'
} }
def versionFile = 'src/main/resources/ragConnectVersion.properties'
def oldProps = new Properties()
try {
file(versionFile).withInputStream { stream -> oldProps.load(stream) }
version = oldProps['version']
} catch (e) {
// this happens, if either the properties file is not present, or cannot be read from
throw new GradleException("File ${versionFile} not found or unreadable. Aborting.", e)
}
task newVersion() {
doFirst {
def props = new Properties()
props['version'] = value
props.store(file(versionFile).newWriter(), null)
}
}
sourceSets { sourceSets {
main { main {
java.srcDir "src/gen/java" java.srcDir "src/gen/java"
......
...@@ -9,31 +9,11 @@ aspect Navigation { ...@@ -9,31 +9,11 @@ aspect Navigation {
eq RagConnect.getChild().ragconnect() = this; eq RagConnect.getChild().ragconnect() = this;
eq MRagConnect.getChild().ragconnect() = getRagConnect(); eq MRagConnect.getChild().ragconnect() = getRagConnect();
// --- containedFile (first equation should be in preprocessor) --- // --- containedFile
eq Program.getChild().containedFile() = null;
eq RagConnect.getChild().containedFile() = null; eq RagConnect.getChild().containedFile() = null;
// --- isTypeComponent (should be in preprocessor) --- // --- containedFileName ---
syn boolean Component.isTypeComponent() = false;
eq TypeComponent.isTypeComponent() = true;
// --- asTypeComponent (should be in preprocessor) ---
syn TypeComponent Component.asTypeComponent() = null;
eq TypeComponent.asTypeComponent() = this;
// --- isListComponent (should be in preprocessor) ---
syn boolean Component.isListComponent() = false;
eq ListComponent.isListComponent() = true;
// --- asListComponent (should be in preprocessor) ---
syn ListComponent Component.asListComponent() = null;
eq ListComponent.asListComponent() = this;
// --- containedFileName (should replace containedFile in preprocessor) ---
inh String ASTNode.containedFileName();
eq GrammarFile.getChild().containedFileName() = getFileName();
eq RagConnect.getChild().containedFileName() = getFileName(); eq RagConnect.getChild().containedFileName() = getFileName();
eq Program.getChild().containedFileName() = null;
eq MRagConnect.getChild().containedFileName() = null; eq MRagConnect.getChild().containedFileName() = null;
// --- isTokenUpdateDefinition --- // --- isTokenUpdateDefinition ---
...@@ -71,10 +51,4 @@ aspect Navigation { ...@@ -71,10 +51,4 @@ aspect Navigation {
// --- isDefaultMappingDefinition --- // --- isDefaultMappingDefinition ---
syn boolean MappingDefinition.isDefaultMappingDefinition() = false; syn boolean MappingDefinition.isDefaultMappingDefinition() = false;
eq DefaultMappingDefinition.isDefaultMappingDefinition() = true; eq DefaultMappingDefinition.isDefaultMappingDefinition() = true;
// --- allTokenComponents ---
coll java.util.Set<TokenComponent> Program.allTokenComponents() [new java.util.HashSet<>()] root Program;
TokenComponent contributes this
to Program.allTokenComponents()
for program();
} }
...@@ -282,6 +282,7 @@ aspect RelationGeneration { ...@@ -282,6 +282,7 @@ aspect RelationGeneration {
right.setType(getSource().containingTypeDecl()); right.setType(getSource().containingTypeDecl());
result.setLeft(left); result.setLeft(left);
result.setRight(right); result.setRight(right);
result.addComment(new WhitespaceComment("\n"));
return result; return result;
} }
} }
......
...@@ -13,5 +13,8 @@ import org.jastadd.ragconnect.parser.RagConnectParser.Terminals; ...@@ -13,5 +13,8 @@ import org.jastadd.ragconnect.parser.RagConnectParser.Terminals;
%yylexthrow beaver.Scanner.Exception %yylexthrow beaver.Scanner.Exception
%scanerror RagConnectScanner.ScannerError %scanerror RagConnectScanner.ScannerError
%x COMMENT
%s DECLARATION
%line %line
%column %column
"read" { return sym(Terminals.READ); } "read" { yybegin(DECLARATION); return sym(Terminals.READ); }
"write" { return sym(Terminals.WRITE); } "write" { yybegin(DECLARATION); return sym(Terminals.WRITE); }
"using" { return sym(Terminals.USING); } "using" { return sym(Terminals.USING); }
"canDependOn" { return sym(Terminals.CAN_DEPEND_ON); } "canDependOn" { return sym(Terminals.CAN_DEPEND_ON); }
"maps" { return sym(Terminals.MAPS); } "maps" { return sym(Terminals.MAPS); }
......
package org.jastadd.ragconnect.compiler; package org.jastadd.ragconnect.compiler;
import beaver.Parser; import beaver.Parser;
import org.jastadd.ragconnect.compiler.options.CommandLine; import org.jastadd.option.BooleanOption;
import org.jastadd.ragconnect.compiler.options.FlagOption; import org.jastadd.option.ValueOption;
import org.jastadd.ragconnect.compiler.options.Option; import org.jastadd.relast.compiler.AbstractCompiler;
import org.jastadd.ragconnect.compiler.options.StringOption; import org.jastadd.relast.compiler.CompilerException;
import org.jastadd.ragconnect.ast.*; import org.jastadd.ragconnect.ast.*;
import org.jastadd.ragconnect.parser.RagConnectParser; import org.jastadd.ragconnect.parser.RagConnectParser;
import org.jastadd.ragconnect.scanner.RagConnectScanner; import org.jastadd.ragconnect.scanner.RagConnectScanner;
import java.io.*; import java.io.*;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.nio.file.StandardCopyOption; import java.nio.file.StandardCopyOption;
import java.util.ArrayList; import java.util.Collection;
import java.util.List; import java.util.MissingResourceException;
import java.util.ResourceBundle;
public class Compiler { public class Compiler extends AbstractCompiler {
private StringOption optionOutputDir; // private ValueOption optionOutputDir;
private StringOption optionInputGrammar; private ValueOption optionRootNode;
private StringOption optionRootNode; private BooleanOption optionVerbose;
private StringOption optionInputRagConnect; private BooleanOption optionLogReads;
private FlagOption optionHelp; private BooleanOption optionLogWrites;
private FlagOption optionVerbose;
private FlagOption optionLogReads;
private FlagOption optionLogWrites;
private ArrayList<Option<?>> options;
private CommandLine commandLine;
public Compiler() { public Compiler() {
options = new ArrayList<>(); super("ragconnect", true);
addOptions();
}
public void run(String[] args) throws CommandLine.CommandLineException, CompilerException {
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
System.setProperty("mustache.debug", "true");
options = new ArrayList<>();
addOptions();
commandLine = new CommandLine(options);
commandLine.parse(args);
if (optionHelp.isSet()) {
printUsage();
return;
} }
if (optionVerbose.isSet()) { @Override
try { protected int compile() throws CompilerException {
run(); if (getConfiguration().shouldPrintVersion()) {
} catch (CompilerException e) { System.out.println(readVersion());
e.printStackTrace(); return 0;
throw e;
}
} else {
run();
} }
if (getConfiguration().shouldPrintHelp()) {
getConfiguration().printHelp(System.out);
return 0;
} }
private void run() throws CompilerException { printMessage("Running RagConnect " + readVersion());
String outputDir;
if (optionOutputDir.isSet()) { if (!getConfiguration().outputDir().exists()) {
outputDir = optionOutputDir.getValue();
} else {
outputDir = "gen";
System.out.println("No output output dir is set. Assuming '" + outputDir + "'.");
}
try { try {
Files.createDirectories(Paths.get(outputDir)); Files.createDirectories(getConfiguration().outputDir().toPath());
} catch (IOException e) { } catch (IOException e) {
throw new CompilerException("Error creating output dir " + outputDir, e); throw new CompilerException("Error creating output dir " + getConfiguration().outputDir(), e);
} }
printMessage("Running RagConnect Preprocessor");
if (anyRequiredOptionIsUnset()) {
throw new CompilerException("Aborting due to missing values for required options.");
} }
List<String> otherArgs = commandLine.getArguments(); RagConnect ragConnect = parseProgram(getConfiguration().getFiles());
if (!otherArgs.isEmpty()) {
printMessage("Superfluous arguments will be ignored: " + otherArgs);
}
RagConnect ragConnect = parseProgram(optionInputGrammar.getValue(), optionInputRagConnect.getValue());
if (!ragConnect.errors().isEmpty()) { if (!ragConnect.errors().isEmpty()) {
System.err.println("Errors:"); System.err.println("Errors:");
...@@ -103,114 +70,166 @@ public class Compiler { ...@@ -103,114 +70,166 @@ public class Compiler {
throw new CompilerException("Could not open " + mqttHandlerFileName); throw new CompilerException("Could not open " + mqttHandlerFileName);
} }
Files.copy(inputStream, Files.copy(inputStream,
Paths.get(outputDir, mqttHandlerFileName), getConfiguration().outputDir().toPath().resolve(mqttHandlerFileName),
StandardCopyOption.REPLACE_EXISTING); StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) { } catch (IOException e) {
throw new CompilerException("Could not copy " + mqttHandlerFileName, e); throw new CompilerException("Could not copy " + mqttHandlerFileName, e);
} }
writeToFile(outputDir + "/Grammar.relast", ragConnect.getProgram().generateAbstractGrammar()); for (GrammarFile grammarFile : ragConnect.getProgram().getGrammarFileList()) {
writeToFile(outputDir + "/RagConnect.jadd", ragConnect.generateAspect(optionRootNode.getValue())); Path outputFile = getConfiguration().outputDir().toPath().resolve(grammarFile.getFileName());
} writeToFile(outputFile, grammarFile.generateAbstractGrammar());
private boolean anyRequiredOptionIsUnset() {
boolean foundError = false;
for (Option<?> option : options) {
if (option.hasArgument() == Option.HasArgument.YES && !option.isSet()) {
System.err.println("Option '" + option.getName() +
"' (" + option.getDescription() + ") is required but unset!");
foundError = true;
} }
writeToFile(getConfiguration().outputDir().toPath().resolve("RagConnect.jadd"), ragConnect.generateAspect(optionRootNode.value()));
return 0;
} }
return foundError;
}
public static void main(String[] args) { public static void main(String[] args) {
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
System.setProperty("mustache.debug", "true");
try { try {
new Compiler().run(args); new Compiler().run(args);
} catch (CommandLine.CommandLineException | CompilerException e) { } catch (CompilerException e) {
System.err.println(e.getMessage()); System.err.println(e.getMessage());
System.exit(-1); System.exit(-1);
} }
} }
/**
* Reads the version string.
*
* The version string is read from the property file
* src/main/resources/Version.properties. This
* file should be generated during the build process. If it is missing
* then there is some problem in the build script.
*
* @author Jesper Öqvist <jesper.oqvist@cs.lth.se>
* @return the read version string, or <code>version ?</code>
*/
private String readVersion() {
try {
ResourceBundle resources = ResourceBundle.getBundle("ragConnectVersion");
return resources.getString("version");
} catch (MissingResourceException e) {
return "version ?";
}
}
private void printMessage(String message) { private void printMessage(String message) {
System.out.println(message); System.out.println(message);
} }
private void writeToFile(String filename, String str) throws CompilerException { private void writeToFile(Path path, String str) throws CompilerException {
try { try (BufferedWriter writer = Files.newBufferedWriter(path)) {
PrintWriter writer = new PrintWriter(filename); writer.append(str);
writer.print(str);
writer.close();
} catch (Exception e) { } catch (Exception e) {
throw new CompilerException("Could not write to file " + filename, e); throw new CompilerException("Could not write to file " + path.toAbsolutePath(), e);
} }
} }
private void addOptions() { protected void initOptions() {
optionOutputDir = addOption(new StringOption("outputDir", "target directory for the generated files.")); super.initOptions();
optionInputGrammar = addOption(new StringOption("inputGrammar", "base grammar.")); optionRootNode = addOption(
optionRootNode = addOption(new StringOption("rootNode", "root node in the base grammar.")); new ValueOption("rootNode", "root node in the base grammar.")
optionInputRagConnect = addOption(new StringOption("inputRagConnect", "connect definition file.")); .acceptAnyValue()
optionHelp = addOption(new FlagOption("help", "Print usage and exit.")); .needsValue(true));
optionVerbose = addOption(new FlagOption("verbose", "Print more messages while compiling.")); optionVerbose = addOption(
optionLogReads = addOption(new FlagOption("logReads", "Enable logging for every read.")); new BooleanOption("verbose", "Print more messages while compiling.")
optionLogWrites = addOption(new FlagOption("logWrites", "Enable logging for every write.")); .defaultValue(false));
} optionLogReads = addOption(
new BooleanOption("logReads", "Enable logging for every read.")
private <OptionType extends Option<?>> OptionType addOption(OptionType option) { .defaultValue(false));
options.add(option); optionLogWrites = addOption(
return option; new BooleanOption("logWrites", "Enable logging for every write.")
.defaultValue(false));
} }
private RagConnect parseProgram(String inputGrammarFileName, String inputRagConnectFileName) throws CompilerException { private RagConnect parseProgram(Collection<String> files) throws CompilerException {
Program program = new Program(); Program program = new Program();
RagConnect ros2Rag;
GrammarFile inputGrammar;
try (BufferedReader reader = Files.newBufferedReader(Paths.get(inputGrammarFileName))) { RagConnect ragConnect = new RagConnect();
ragConnect.setProgram(program);
GrammarFile ragConnectGrammarPart = new GrammarFile();
ragConnectGrammarPart.setFileName("RagConnect.relast");
program.addGrammarFile(ragConnectGrammarPart);
for (String filename : files) {
String extension = filename.substring(filename.lastIndexOf('.') + 1);
switch (extension) {
case "ast":
case "relast":
// processGrammar
parseGrammar(program, filename);
break;
case "connect":
// process ragConnect
RagConnect parsedRagConnect = parseRagConnect(program, filename);
mergeRagConnectDefinitions(ragConnect, parsedRagConnect);
break;
default:
throw new CompilerException("Unknown file extension in " + filename);
}
}
ragConnect.treeResolveAll();
ragConnect.additionalRelations().forEach(ragConnectGrammarPart::addDeclaration);
ASTNode.loggingEnabledForReads = optionLogReads.value();
ASTNode.loggingEnabledForWrites = optionLogWrites.value();
return ragConnect;
}
private void parseGrammar(Program program, String filename) throws CompilerException {
try (BufferedReader reader = Files.newBufferedReader(Paths.get(filename))) {
RagConnectScanner scanner = new RagConnectScanner(reader); RagConnectScanner scanner = new RagConnectScanner(reader);
RagConnectParser parser = new RagConnectParser(); RagConnectParser parser = new RagConnectParser();
inputGrammar = (GrammarFile) parser.parse(scanner); GrammarFile grammarFile = (GrammarFile) parser.parse(scanner);
if (optionVerbose.isSet()) { if (optionVerbose.value()) {
inputGrammar.dumpTree(System.out); grammarFile.dumpTree(System.out);
} }
program.addGrammarFile(inputGrammar); program.addGrammarFile(grammarFile);
inputGrammar.treeResolveAll(); grammarFile.treeResolveAll();
inputGrammar.setFileName(inputGrammarFileName); grammarFile.setFileName(toBaseName(filename));
} catch (IOException | Parser.Exception e) { } catch (IOException | Parser.Exception e) {
throw new CompilerException("Could not parse grammar file " + inputGrammarFileName, e); throw new CompilerException("Could not parse grammar file " + filename, e);
}
} }
try (BufferedReader reader = Files.newBufferedReader(Paths.get(inputRagConnectFileName))) { private RagConnect parseRagConnect(Program program, String filename) throws CompilerException {
try (BufferedReader reader = Files.newBufferedReader(Paths.get(filename))) {
RagConnectScanner scanner = new RagConnectScanner(reader); RagConnectScanner scanner = new RagConnectScanner(reader);
RagConnectParser parser = new RagConnectParser(); RagConnectParser parser = new RagConnectParser();
ros2Rag = (RagConnect) parser.parse(scanner, RagConnectParser.AltGoals.ragconnect); RagConnect ragConnect = (RagConnect) parser.parse(scanner, RagConnectParser.AltGoals.ragconnect);
ros2Rag.setProgram(program); ragConnect.setProgram(program);
ros2Rag.setFileName(inputRagConnectFileName); ragConnect.setFileName(toBaseName(filename));
return ragConnect;
} catch (IOException | Parser.Exception e) { } catch (IOException | Parser.Exception e) {
throw new CompilerException("Could not parse ros2rag file " + inputRagConnectFileName, e); throw new CompilerException("Could not parse connect file " + filename, e);
} }
ros2Rag.treeResolveAll();
ros2Rag.additionalRelations().forEach(inputGrammar::addDeclaration);
ASTNode.loggingEnabledForReads = optionLogReads.isSet();
ASTNode.loggingEnabledForWrites = optionLogWrites.isSet();
return ros2Rag;
} }
protected void printUsage() { /**
System.out.println("Usage: java -jar ros2rag.jar [--option1] [--option2=value] ... <filename1> <filename2> ... "); * Extracts the basename of the given file, with file extension
System.out.println("Options:"); * @param filename the given filename
System.out.print(commandLine.printOptionHelp()); * @return the basename
*/
private String toBaseName(String filename) {
return new File(filename).getName();
} }
public static class CompilerException extends Exception { private void mergeRagConnectDefinitions(RagConnect ragConnect, RagConnect ragConnectToIntegrate) {
CompilerException(String message) { for (UpdateDefinition updateDefinition : ragConnectToIntegrate.getUpdateDefinitionList()) {
super(message); ragConnect.addUpdateDefinition(updateDefinition);
}
for (MappingDefinition mappingDefinition : ragConnectToIntegrate.getMappingDefinitionList()) {
ragConnect.addMappingDefinition(mappingDefinition);
} }
CompilerException(String message, Throwable cause) { for (DependencyDefinition dependencyDefinition : ragConnectToIntegrate.getDependencyDefinitionList()) {
super(message, cause); ragConnect.addDependencyDefinition(dependencyDefinition);
} }
} }
// protected void printUsage() {
// System.out.println("Usage: java -jar ragconnect.jar [--option1] [--option2=value] ... <filename1> <filename2> ... ");
// System.out.println("Options:");
// System.out.print(commandLine.printOptionHelp());
// }
} }
package org.jastadd.ragconnect.compiler.options;
import java.util.*;
public class CommandLine {
private final Collection<Option<?>> options;
private final Map<String, Option<?>> mapping;
private final List<String> arguments;
public CommandLine(Collection<Option<?>> options) {
this.options = options;
this.mapping = new HashMap<>();
for (Option<?> option : options) {
mapping.put(option.getName(), option);
}
this.arguments = new ArrayList<>();
}
public void parse(String[] args) throws CommandLineException {
int i = 0;
while (i < args.length) {
if (args[i].startsWith(Option.PREFIX)) {
int argumentIndex = args[i].indexOf("=");
String name;
String argument = null;
if (argumentIndex > 0) {
name = args[i].substring(2, argumentIndex);
argument = args[i].substring(argumentIndex + 1);
} else {
name = args[i].substring(2);
}
Option<?> option = mapping.get(name);
if (option == null) {