Skip to content

Commit

Permalink
Remove cf primitives and extension via autoconverter
Browse files Browse the repository at this point in the history
  • Loading branch information
@qiemem @LaCuneta
qiemem authored and LaCuneta committed Apr 17, 2019
1 parent dd9382e commit aadad64
Show file tree
Hide file tree
Showing 4 changed files with 122 additions and 45 deletions.
12 changes: 11 additions & 1 deletion netlogo-core/src/main/fileformat/AutoConversionList.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,16 @@ object AutoConversionList {
},
"NetLogo 6.0-BETA2" -> {
changeAllCode("rename range to _range", Seq(_.replaceToken("range", "_range")), Seq("range"))
},
"NetLogo 6.1" -> {
changeAllCode("remove CF extension; replace with build-in primitives",
Seq(
_.removeExtension("cf"),
_.replaceToken("cf:ifelse", "ifelse"),
_.replaceToken("cf:ifelse-value", "ifelse-value"),
),
Seq("cf")
)
}
)
)
}
1 change: 1 addition & 0 deletions parser-core/src/main/core/SourceRewriter.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package org.nlogo.core
trait SourceRewriter {
def addGlobal(global: String): String
def addExtension(extension: String): String
def removeExtension(extension: String): String
def addReporterProcedure(name: String, args: Seq[String], body: String): String
def remove(commandName: String): String
def replaceToken(originalToken: String, replaceToken:String): String
Expand Down
138 changes: 94 additions & 44 deletions parser-core/src/main/parse/AstRewriter.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,22 @@

package org.nlogo.parse

import org.nlogo.core.{ CompilationOperand, Femto, ProcedureDefinition,
ProcedureSyntax, SourceLocation, SourceRewriter, StructureResults, TokenizerInterface }
import org.nlogo.core.{
CompilationOperand,
Femto,
ProcedureDefinition,
ProcedureSyntax,
SourceLocation,
SourceRewriter,
StructureResults,
Token,
TokenType,
TokenizerInterface
}

import scala.util.matching.Regex

class AstRewriter(val tokenizer: TokenizerInterface, op: CompilationOperand)
extends SourceRewriter
with NetLogoParser {
class AstRewriter(val tokenizer: TokenizerInterface, op: CompilationOperand) extends SourceRewriter with NetLogoParser {

def preserveBody(structureResults: StructureResults, header: String, procedures: String, footer: String): String =
header + procedures + footer
Expand Down Expand Up @@ -38,7 +46,7 @@ class AstRewriter(val tokenizer: TokenizerInterface, op: CompilationOperand)
rewrite(folder, preserveBody _)
}

def replaceToken(original: String, replacement:String): String = {
def replaceToken(original: String, replacement: String): String = {
val source = op.sources("")
val tokens = tokenizer.tokenizeString(source)
val buf = new StringBuilder(source)
Expand All @@ -54,8 +62,47 @@ class AstRewriter(val tokenizer: TokenizerInterface, op: CompilationOperand)
rewrite(NoopFolder, declarationReplace("extensions", extensionsRegex, _.extensions.map(_.text), newExtension))
}

def removeExtension(extension: String): String = {
import TokenType.{OpenBracket, CloseBracket, Ident}
val extValue = extension.toUpperCase
val source = op.sources("")
val tokens = tokenizer.tokenizeString(source).toStream
val buf = new StringBuilder(source)

def extensions(tokens: Stream[Token]): Seq[Token] = tokens match {
case Token(_, Ident, "EXTENSIONS") #:: Token(_, OpenBracket, _) #:: rest => {
val (exts, after) = rest.span(_.tpe == Ident)
tokens.take(2) ++ exts ++ after.take(1)
}
case _ #:: rest => extensions(rest)
case Stream.Empty => Seq.empty[Token]
}

extensions(tokens) match {
case extKeyword +: Token(_, OpenBracket, _)
+: Token(_, Ident, extName)
+: (close @ Token(_, CloseBracket, _))
+: Nil if extName == extValue =>
// eat empty lines
val end = (close.end until source.length).find(_ == '\n').getOrElse(source.length)
buf.replace(extKeyword.start, end, "")
case extSection =>
extSection.span(_.value != extValue) match {
case (_ :+ precedingToken, extToken +: _) =>
// This basically preserves formatting exactly. So
// extensions [ abc def ] => extensions [ abc ]
// extensions [abc def] => extension [abc]
// It just makes whatever comes after the extension come after the preceding extension instead.
buf.replace(precedingToken.end, extToken.end, "")
case _ => // not found
}
}
buf.toString
}

def addGlobal(newGlobal: String): String = {
rewrite(NoopFolder, declarationReplace("globals", globalsRegex, _.program.userGlobals.map(_.toLowerCase), newGlobal))
rewrite(NoopFolder,
declarationReplace("globals", globalsRegex, _.program.userGlobals.map(_.toLowerCase), newGlobal))
}

def addReporterProcedure(name: String, args: Seq[String], body: String): String = {
Expand All @@ -72,34 +119,34 @@ class AstRewriter(val tokenizer: TokenizerInterface, op: CompilationOperand)
private val globalsRegex = new Regex("(?i)(?m)globals\\s+\\[[^]]*\\]")

private def declarationReplace(
declKeyword: String,
declRegex: Regex,
declItems: StructureResults => Seq[String],
addedItem: String)(
res: StructureResults, headers: String, procedures: String, footer: String): String = {
declKeyword: String,
declRegex: Regex,
declItems: StructureResults => Seq[String],
addedItem: String)(res: StructureResults, headers: String, procedures: String, footer: String): String = {
val newDecl =
declKeyword + " " + (declItems(res) :+ addedItem).distinct.mkString("[", " ", "]")
val modifiedHeaders = declRegex
.findFirstMatchIn(headers)
.map(m => headers.take(m.start) + newDecl + headers.drop(m.end))
.getOrElse(newDecl+ "\n" + headers)
.getOrElse(newDecl + "\n" + headers)
modifiedHeaders + procedures + footer
}

private def procedureAdd(content: String)(
res: StructureResults, headers: String, procedures: String, footer: String): String = {
headers + procedures + "\n" + content + "\n" + footer
private def procedureAdd(
content: String)(res: StructureResults, headers: String, procedures: String, footer: String): String = {
headers + procedures + "\n" + content + "\n" + footer
}

def rewrite(
visitor: PositionalAstFolder[AstEdit],
wholeFile: (StructureResults, String, String, String) => String,
sourceName: String = ""): String = {
def rewrite(visitor: PositionalAstFolder[AstEdit],
wholeFile: (StructureResults, String, String, String) => String,
sourceName: String = ""): String = {

val (procs, structureResults) = basicParse(op)

def getSource(filename: String): String =
op.sources.get(filename).orElse(IncludeFile(op.compilationEnvironment, filename).map(_._2))
op.sources
.get(filename)
.orElse(IncludeFile(op.compilationEnvironment, filename).map(_._2))
.getOrElse(throw new Exception("Unable to find file: " + filename))

val (wsMap, fileHeaders, fileFooters) = trackWhitespace(getSource _, procs)
Expand All @@ -120,11 +167,14 @@ class AstRewriter(val tokenizer: TokenizerInterface, op: CompilationOperand)
rewritten.lines.map(eolWhitespace.replaceAllIn(_, "")).mkString("\n")
}

def trackWhitespace(getSource: String => String, procs: Iterable[ProcedureDefinition]): (WhitespaceMap, Map[String, String], Map[String, String]) = {
def trackWhitespace(
getSource: String => String,
procs: Iterable[ProcedureDefinition]): (WhitespaceMap, Map[String, String], Map[String, String]) = {
val ws = new WhiteSpace.Tracker(getSource, tokenizer)
var fileHeaders: Map[String, String] = Map()
var fileFooters: Map[String, String] = Map()
var procedurePositions: Map[String, Map[String, ProcedureSyntax]] = Map()

def procedurePosition(file: String, procedureName: String): ProcedureSyntax = {
if (procedurePositions.isDefinedAt(file))
procedurePositions(file)(procedureName)
Expand All @@ -147,36 +197,36 @@ class AstRewriter(val tokenizer: TokenizerInterface, op: CompilationOperand)

val whiteSpaces =
procs.foldLeft((WhitespaceMap.empty, WhiteSpace.Context.empty)) {
case ((whitespaceLog, ctx), proc) =>
val procSyntax = procedurePosition(proc.filename, proc.procedure.name)
val newContext =
if (ctx.lastPosition.map(_._2.filename).contains(proc.filename)) {
WhiteSpace.Context.empty(ctx.lastPosition)
} else {
val procStart = procSyntax.declarationKeyword.start
fileHeaders = (fileHeaders + (proc.filename -> getSource(proc.filename).slice(0, procStart)))
WhiteSpace.Context.empty(Some((AstPath(), SourceLocation(procStart, procStart, proc.filename))))
}
val r = ws.visitProcedureDefinition(proc)(newContext)
addTrailingWhitespace(r)
(whitespaceLog ++ r.whitespaceLog, r)
}
case ((whitespaceLog, ctx), proc) =>
val procSyntax = procedurePosition(proc.filename, proc.procedure.name)
val newContext =
if (ctx.lastPosition.map(_._2.filename).contains(proc.filename)) {
WhiteSpace.Context.empty(ctx.lastPosition)
} else {
val procStart = procSyntax.declarationKeyword.start
fileHeaders = (fileHeaders + (proc.filename -> getSource(proc.filename).slice(0, procStart)))
WhiteSpace.Context.empty(Some((AstPath(), SourceLocation(procStart, procStart, proc.filename))))
}
val r = ws.visitProcedureDefinition(proc)(newContext)
addTrailingWhitespace(r)
(whitespaceLog ++ r.whitespaceLog, r)
}
if (procs.isEmpty) {
fileHeaders = fileHeaders + ("" -> getSource(""))
}
addTrailingWhitespace(whiteSpaces._2)
(whiteSpaces._1, fileHeaders, fileFooters)
}

def format(
edit: AstEdit,
procs: Iterable[ProcedureDefinition]): String = {
import edit.{ operations, wsMap }
def format(edit: AstEdit, procs: Iterable[ProcedureDefinition]): String = {
import edit.{operations, wsMap}
val formatter = new Formatter
val res = procs.filter(p => op.sources.contains(p.procedure.filename)).foldLeft[AstFormat](Formatter.context("", operations, wsMap = wsMap)) {
case (acc, proc) =>
formatter.visitProcedureDefinition(proc)(Formatter.context(acc.text, acc.operations, wsMap = wsMap))
}
val res = procs
.filter(p => op.sources.contains(p.procedure.filename))
.foldLeft[AstFormat](Formatter.context("", operations, wsMap = wsMap)) {
case (acc, proc) =>
formatter.visitProcedureDefinition(proc)(Formatter.context(acc.text, acc.operations, wsMap = wsMap))
}
res.text
}
}
16 changes: 16 additions & 0 deletions parser-jvm/src/test/parse/AstRewriterTests.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,19 @@ class AstRewriterTests extends FunSuite {
assertResult("extensions []\nglobals [x] to bar end")(addExtension("globals [x] to bar end", ""))
}

test("remove extension") {
assertResult("")(removeExtension("extensions [abc]", "abc"))
assertResult("extensions [ abc ]")(removeExtension("extensions [ abc def ]", "def"))
assertResult("extensions [ def ]")(removeExtension("extensions [ abc def ]", "abc"))
assertResult("extensions [ abc ghi ]")(removeExtension("extensions [ abc def ghi ]", "def"))
assertResult("")(removeExtension("extensions [\n abc\n]", "abc"))
assertResult("extensions [\n abc\n]")(removeExtension("extensions [\n abc\n def\n]", "def"))
assertResult("extensions [\n def\n]")(removeExtension("extensions [\n abc\n def\n]", "abc"))

assertResult("extensions []")(removeExtension("extensions []", "abc"))
assertResult("extensions [ def ]")(removeExtension("extensions [ def ]", "abc"))
}

test("add global") {
assertResult("globals [foo]")(addGlobal("", "foo"))
assertResult("globals [foo]\nto bar end")(addGlobal("to bar end", "foo"))
Expand Down Expand Up @@ -191,6 +204,9 @@ class AstRewriterTests extends FunSuite {
added.trim
}

def removeExtension(source:String, extension: String): String =
rewriter(source).removeExtension(extension).trim

def addGlobal(source: String, global: String): String = {
val added = rewriter(source).addGlobal(global)
added.trim
Expand Down

0 comments on commit aadad64

Please sign in to comment.