Skip to content
Snippets Groups Projects
Commit c502ab97 authored by Tim Kluge's avatar Tim Kluge
Browse files

Initial commit

parents
No related branches found
No related tags found
No related merge requests found
Pipeline #9722 passed
package jackrat.de.timklge.jackrat
import de.timklge.jackrat.Node
import de.timklge.jackrat.Parser
@OptIn(ExperimentalStdlibApi::class)
private fun getBreaks(input: String) = buildSet {
val nonReadable = Regex("""[^\p{L}\p{N}\p{Pc}]""").findAll(input).map { it.range.first }.toSet()
var previousWord = false
input.forEachIndexed { index, _ ->
val currentWord = !nonReadable.contains(index)
if(!currentWord || !previousWord) add(index)
previousWord = currentWord
}
add(input.count())
}
data class Scanner(val input: String, var position: Int, private val skipWhitespace: Boolean,
val memoization: MutableMap<Int, MutableMap<Parser, MemoEntry>>,
val heads: MutableMap<Int, Head>,
var invocationStack: Lr?,
val skipRegex: Regex?,
val breaks: Set<Int>) {
constructor(input: String, position: Int = 0, skipWhitespace: Boolean = true) : this(input, position, skipWhitespace,
mutableMapOf(), mutableMapOf(), null, if(skipWhitespace) Regex("""^[\r\n\t ]+""") else null, getBreaks(input)
)
constructor(input: String, position: Int = 0, skip: Regex) : this(input, position, true, mutableMapOf(), mutableMapOf(), null, skip, getBreaks(input))
fun match(regex: Regex): String? {
val matched = regex.find(input.substring(position))
return if(matched != null){
position += matched.value.length
return matched.value
} else null
}
fun skipWhitespace() {
if(skipRegex != null) match(skipRegex)
}
fun isAtBreak(): Boolean {
return breaks.contains(position)
}
fun recall(rule: Parser, pos: Int): MemoEntry? {
val mmap: MutableMap<Parser, MemoEntry>? = memoization.getOrElse(pos, { null })
val m: MemoEntry? = if(mmap != null) mmap[rule] else null
val head = heads.getOrElse(pos, { return m })
// Do not evaluate any rule that is not involved in this left recursion
if(m == null && !head.involvedSet.contains(rule)) return MemoEntry(null, null, position)
if(head.evalSet.contains(rule)){
head.evalSet.remove(rule)
val result = rule.Match(this)
return MemoEntry(null, result, position)
}
return m
}
fun setupLr(rule: Parser, l: Lr) {
if(l.head == null) l.head = Head(rule)
var stack = invocationStack
while(stack != null && stack.head != l.head){
stack.head = l.head
val newInvolved = mutableMapOf<Parser, Boolean>()
l.head!!.involvedSet.forEach { newInvolved[it.key] = true }
newInvolved[stack.rule] = true
l.head!!.involvedSet = newInvolved
stack = stack.next
}
}
fun growLr(rule: Parser, p: Int, m: MemoEntry, h: Head): Node? {
heads[p] = h
while(true){
position = p
h.evalSet = mutableMapOf()
h.involvedSet.forEach { h.evalSet[it.key] = it.value }
val result = rule.Match(this)
if(result == null || position <= m.Position) break
m.lr = null
m.Ans = result
m.Position = position
}
heads.remove(p)
position = m.Position
return m.Ans
}
fun lrAnswer(rule: Parser, pos: Int, m: MemoEntry): Node? {
val h = m.lr!!.head!!
if(h.rule != rule) return m.lr!!.seed
m.Ans = m.lr!!.seed
m.lr = null
if(m.Ans == null) return null
return growLr(rule, pos, m, h)
}
fun applyRule(rule: Parser): Node? {
val startPosition = position
val mmap: MutableMap<Parser, MemoEntry> = if(memoization.contains(startPosition)) memoization[startPosition]!! else {
val newmap = mutableMapOf<Parser, MemoEntry>()
memoization[startPosition] = newmap
newmap
}
val m = recall(rule, startPosition)
if(m == null){
val lr = Lr(null, rule, null, invocationStack)
invocationStack = lr
val memo = MemoEntry(lr, null, startPosition)
mmap[rule] = memo
val ans = rule.Match(this)
invocationStack = invocationStack!!.next
memo.Position = position
if(lr.head != null){
lr.seed = ans
return lrAnswer(rule, startPosition, memo)
}
memo.lr = null
memo.Ans = ans
return ans
}
position = m.Position
if(m.lr != null){
setupLr(rule, m.lr!!)
return m.lr!!.seed
}
return m.Ans
}
}
data class Head(val rule: Parser, var involvedSet: MutableMap<Parser, Boolean>, var evalSet: MutableMap<Parser, Boolean>){
constructor(rule: Parser) : this(rule, mutableMapOf(), mutableMapOf())
}
data class Lr(var seed: Node?, val rule: Parser, var head: Head?, val next: Lr?)
data class MemoEntry(var lr: Lr?, var Ans: Node?, var Position: Int)
\ No newline at end of file
package jackrat
import de.timklge.jackrat.AndParser
import de.timklge.jackrat.AtomParser
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
import kotlin.test.fail
class AndOrTests {
@Test
fun TestAndInsensitive() {
val input = "HELLO world"
val scanner = Scanner(input)
val helloParser = AtomParser("Hello", true)
val worldParser = AtomParser("World", true)
val helloAndWorldParser = AndParser(listOf(helloParser, worldParser))
val node = helloAndWorldParser.parse(scanner)
if(node.parser != helloAndWorldParser) fail("And combinator creates node with wrong parser")
if(node.matched != "HELLOworld") fail("And combinator doesn't match complete input")
}
}
\ No newline at end of file
package jackrat
import de.timklge.jackrat.AndParser
import de.timklge.jackrat.AtomParser
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
import kotlin.test.assertEquals
class AndTests {
@Test
fun TestAndInsensitive(){
val input = "HELLO world"
val scanner = Scanner(input)
val helloParser = AtomParser("Hello", true)
val worldParser = AtomParser("World", true)
val helloAndWorldParser = AndParser(listOf(helloParser, worldParser))
val node = helloAndWorldParser.parse(scanner)
assertEquals(node.parser, helloAndWorldParser)
assertEquals(2, node.children.count())
}
@Test
fun TestAnd(){
val input = "Hello world"
val scanner = Scanner(input)
val helloParser = AtomParser("Hello", false)
val worldParser = AtomParser("world", false)
val helloAndWorldParser = AndParser(listOf(helloParser, worldParser))
val node = helloAndWorldParser.parse(scanner)
assertEquals(node.parser, helloAndWorldParser)
assertEquals(2, node.children.count())
}
}
\ No newline at end of file
package jackrat
import de.timklge.jackrat.AndParser
import de.timklge.jackrat.AtomParser
import de.timklge.jackrat.EmptyParser
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
import kotlin.test.fail
class EmptyTests {
@Test
fun TestEmpty() {
val input = "Hello"
val scanner = Scanner(input)
val emptyParser = EmptyParser()
val helloParser = AtomParser("Hello")
val emptyAndHelloParser = AndParser(listOf(emptyParser, helloParser))
val node = emptyAndHelloParser.parse(scanner)
if(node.parser != emptyAndHelloParser) fail("Empty Test combinator creates node with wrong parser")
if(node.matched != "Hello") fail("Empty Test combinator doesn't match complete input")
if(node.children.count() != 2) fail("Empty Test combinator child count doesn't match")
if(node.children[0].parser != emptyParser || node.children[1].parser != helloParser) fail("Empty Test combinator children do not match")
}
@Test
fun TestDoubleEmpty(){
val input = ""
val scanner = Scanner(input)
val emptyParser = EmptyParser()
val termParser = AndParser(listOf(emptyParser, emptyParser))
val node = termParser.parse(scanner)
if(node.parser != termParser) fail("Double empty parser creates node with wrong parser")
}
}
\ No newline at end of file
package jackrat
import de.timklge.jackrat.AndParser
import de.timklge.jackrat.AtomParser
import de.timklge.jackrat.EndParser
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
import kotlin.test.fail
class EndTests {
@Test
fun EndTest(){
val input = "Hello"
val scanner = Scanner(input)
val endParser = EndParser(false)
val helloParser = AtomParser("Hello")
val helloEndParser = AndParser(listOf(helloParser, endParser))
val node = helloEndParser.parse(scanner)
if(node.parser != helloEndParser) fail("End test combinator creates node with wrong parser")
if(node.matched != input) fail("End test combinator doesn't match complete input")
if(node.children.count() != 2) fail("End test combinator doesn't produce 3 children")
}
}
\ No newline at end of file
package jackrat
import de.timklge.jackrat.*
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
class JsonTests {
@Test
fun TestJSON(){
val input = """
{"menu": {
"header": "SVG Viewer",
"items": [
{"id": "Open"},
{"id": "OpenNew", "label": "Open New"},
null,
{"id": "ZoomIn", "label": "Zoom In"},
{"id": "ZoomOut", "label": "Zoom Out"},
{"id": "OriginalView", "label": "Original View"},
null,
{"id": "Quality"},
{"id": "Pause"},
{"id": "Mute"},
null,
{"id": "Find", "label": "Find..."},
{"id": "FindAgain", "label": "Find Again"},
{"id": "Copy"},
{"id": "CopyAgain", "label": "Copy Again"},
{"id": "CopySVG", "label": "Copy SVG"},
{"id": "ViewSVG", "label": "View SVG"},
{"id": "ViewSource", "label": "View Source"},
{"id": "SaveAs", "label": "Save As"},
null,
{"id": "Help"},
{"id": "About", "label": "About Adobe CVG Viewer..."}
]
}}
""".trimIndent()
val scanner = Scanner(input)
val stringParser = AndParser(listOf(AtomParser("\""), RegexParser("""(?:[^"\\]|\\.)*"""), AtomParser("\"")))
val valueParser = OrParser(listOf())
val propParser = AndParser(listOf(stringParser, AtomParser(":"), valueParser))
val objParser = AndParser(listOf(AtomParser("{"), KleeneParser(propParser, AtomParser(",")), AtomParser("}")))
val nullParser = AtomParser("null")
val numParser = RegexParser("""-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?""")
val boolParser = RegexParser("(true|false)")
val arrayParser = AndParser(listOf(AtomParser("["), KleeneParser(valueParser, AtomParser(",")), AtomParser("]")))
valueParser.children = listOf(nullParser, objParser, stringParser, numParser, boolParser, arrayParser)
valueParser.parse(scanner)
}
}
\ No newline at end of file
package jackrat
import de.timklge.jackrat.AtomParser
import de.timklge.jackrat.KleeneParser
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.fail
class KleeneTests {
@Test
fun KleeneTest(){
val input = "Hello Hello Hello"
val scanner = Scanner(input)
val helloParser = AtomParser("Hello", false)
val helloKleeneParser = KleeneParser(helloParser)
val node = helloKleeneParser.parse(scanner)
assertEquals(helloKleeneParser, node.parser)
assertEquals(3, node.children.count())
}
@Test
fun KleeneIrregularTest(){
val helloParser = AtomParser("Hello")
val irregularInput = "Sonne"
val irregularScanner = Scanner(irregularInput)
val irregularParser = KleeneParser(helloParser)
val irregularNode = irregularParser.parsePartial(irregularScanner)
assertNotNull(irregularNode)
assertEquals(irregularParser, irregularNode.parser, "Irregular kleene parser creates node with wrong parser")
assertEquals(0, irregularNode.children.count(), "Irregular kleene parser doesn't produce zero children for irregular input")
}
@Test
fun KleeneSeparatorTest(){
val input = "Hello, Hello, Hello"
val scanner = Scanner(input)
val helloParser = AtomParser("Hello", false)
val sepParser = AtomParser(",", false)
val helloKleeneParser = KleeneParser(helloParser, sepParser)
val node = helloKleeneParser.parse(scanner)
assertEquals(helloKleeneParser, node.parser, "Kleene combinator creates node with wrong parser")
assertEquals("Hello,Hello,Hello", node.matched, "Kleene combinator doesn't match complete input")
assertEquals(5, node.children.count(), "Kleene combinator doesn't produce five children")
}
}
\ No newline at end of file
package jackrat
import de.timklge.jackrat.*
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
import kotlin.test.assertEquals
class LeftRecursionTest {
@Test
fun TestLeftRecursion(){
val input = "5-1-4-3"
val scanner = Scanner(input)
val emptyParser = EmptyParser()
val emptyParser1 = AndParser(listOf(emptyParser))
val numParser = RegexParser("""\d+""")
val numCombo1 = AndParser(listOf(emptyParser1, emptyParser1, numParser))
val minusParser = AtomParser("-")
val termParser = AndParser(listOf())
val exprParser = OrParser(listOf(termParser, numCombo1))
termParser.children = listOf(exprParser, minusParser, numCombo1)
val node = exprParser.parse(scanner)
assertEquals(node.parser, exprParser)
}
}
\ No newline at end of file
package jackrat
import de.timklge.jackrat.AtomParser
import de.timklge.jackrat.ManyParser
import de.timklge.jackrat.RegexParser
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
class ManyTests {
@Test
fun ManyTest(){
val input = "Hello Hello Hello"
val scanner = Scanner(input)
val helloParser = AtomParser("Hello", false)
val helloManyParser = ManyParser(helloParser)
val node = helloManyParser.parse(scanner)
assertEquals(helloManyParser, node.parser)
assertEquals(3, node.children.count())
}
@Test
fun ManySeparatorTest(){
val input = "Hello, Hello, Hello"
val scanner = Scanner(input)
val helloParser = AtomParser("Hello", false)
val sepParser = AtomParser(",", false)
val helloManyParser = ManyParser(helloParser, sepParser)
val node = helloManyParser.parse(scanner)
assertEquals(helloManyParser, node.parser, "Many combinator creates node with wrong parser")
assertEquals("Hello,Hello,Hello", node.matched, "Many combinator doesn't match complete input")
assertEquals(5, node.children.count(), "Many combinator doesn't produce five children")
}
@Test
fun ManySeparatorRegexTest(){
val input = " 23, 45"
val scanner = Scanner(input)
val digitParser = RegexParser("""\d+""")
val digitSepParser = ManyParser(digitParser, AtomParser(","))
val node = digitSepParser.parse(scanner)
assertEquals(digitSepParser, node.parser, "Many separator combinator creates node with wrong parser")
assertEquals("23,45", node.matched)
assertEquals(3, node.children.count())
}
@Test
fun ManyNoKleeneTest(){
val input = "test"
val scanner = Scanner(input)
val helloParser = AtomParser("Hello")
val node = helloParser.parsePartial(scanner)
assertNull(node)
}
}
\ No newline at end of file
package jackrat
import de.timklge.jackrat.AtomParser
import de.timklge.jackrat.MaybeParser
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
class MaybeTests {
@Test
fun MaybeTest(){
val input = "Hello"
val scanner = Scanner(input)
val helloParser = AtomParser("Hello", false)
val helloMaybeParser = MaybeParser(helloParser)
val node = helloMaybeParser.parse(scanner)
assertEquals(helloMaybeParser, node.parser)
assertEquals(1, node.children.count())
}
@Test
fun MaybeIrregularTest(){
val helloParser = AtomParser("Hello")
val irregularInput = "Sonne"
val irregularScanner = Scanner(irregularInput)
val irregularParser = MaybeParser(helloParser)
val irregularNode = irregularParser.parsePartial(irregularScanner)
assertNotNull(irregularNode)
assertEquals(irregularParser, irregularNode.parser)
assertEquals(0, irregularNode.children.count())
}
}
\ No newline at end of file
package jackrat
import de.timklge.jackrat.AtomParser
import de.timklge.jackrat.OrParser
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
import kotlin.test.assertEquals
class OrTests {
@Test
fun TestOr() {
val input = "World"
val scanner = Scanner(input)
val helloParser = AtomParser("Hello", true)
val worldParser = AtomParser("World", true)
val helloOrWorldParser = OrParser(listOf(helloParser, worldParser))
val node = helloOrWorldParser.parse(scanner)
assertEquals(node.parser, helloOrWorldParser)
assertEquals(1, node.children.count())
}
}
\ No newline at end of file
package jackrat
import de.timklge.jackrat.RegexParser
import jackrat.de.timklge.jackrat.Scanner
import kotlin.test.Test
import kotlin.test.assertNull
class RegexTests {
@Test
fun TestRegex(){
val input = "-3.4"
val scanner = Scanner(input)
val numParser = RegexParser("""-?\d+\.\d+""")
numParser.parse(scanner)
}
@Test
fun TestIrregularRegex(){
val input = "3,4"
val scanner = Scanner(input)
val numParser = RegexParser("""-?\d+\.\d+""")
val node = numParser.parsePartial(scanner)
assertNull(node)
}
}
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment