zodern

Implementing Flare, Part 2

This is the second part in a series of blog posts building the flare Meteor package. This is an experimental build plugin to implement single file Blaze templates and make some other changes to its syntax.

After part one, we are now able to use a single file for the template and any js related to it. In this part, we will allow writing js statements directly in template tags. Instead of creating a helper to do a comparison for an if tag, or to trim whitespace from a string before displaying it, we will be able to put that simple js directly in the template tag, or call functions we've imported or defined in the <script> tag.

Here is an example:


<script>
Template.counter.onCreated(function helloOnCreated() {
this.counter = new ReactiveVar(0);
});

function getCount() {
return Template.instance().counter.get();
}
</script>


<p>Count is even!</p>


<p>My username is .<p>


This is a breaking change, so we have to rethink how spacebars currently works. Spacebars allows accessing variables from the current context, parent context, or to call helpers.

This is how we can write the template, but not what we can pass to the space bars compiler. For the template to be valid, we need to move any js expressions into new helpers, and have the template tag call the helper.

The process will look like this:

  1. Find any template tags in the html AST. They can be attributes, in attribute values, or in text nodes in the AST.
  2. Identify the type of tag. Some tags won't have any js, such as all closing tags, or the exceptions we listed above.
  3. Extract the js expression
  4. Replace the js expression with a call to a helper
  5. Add the helpers to the output js

Let's start with traversing the AST to find where template tags could be.

class Compiler {
constructor() {
this.babelCompiler = new BabelCompiler;
this.babelCompiler.setDiskCacheDirectory(os.tmpdir() + '/meteor-babel-' + Math.random() + Date.now());
}

findTemplateTags(ast) {
const result = [];

function walkChildren(children, handler) {
children.forEach(child => {
handler(child);
if (child.childNodes) {
walkChildren(child.childNodes, handler);
}
});
}

walkChildren(ast.childNodes, node => {
if (node.nodeName === '#text') {
// Scan node.value for template tags
}

node.attrs.forEach(attr => {
if (attr.name.includes('{')) {
// Scan for possible template tag
} else if (typeof attr.value === 'string') {
// Scan for possible template tags
}
});
});

In this, we are walking the AST tree to look at every node, and identifying any text nodes or attributes that could possibly have template tags. The next step is to examine the content to find the tags.

We will write a simple scanner to do this. We could use helpers from the blaze and spacebar packages for this, but they make too many assumptions that are incompatible with some of our plans. Since we aren't compiling the tags or validating them, we don't need a full parser. I implemented a simple scanner class we can use. Put this in a scanner.js file.

const whitespace = /[ \t\r\n]/;

export default class Scanner {
constructor(content, contentContext) {
this.content = content;
this.contentContext = contentContext;
this.index = 0;
}

match (text) {
return this.content.slice(this.index, this.index + text.length) === text;
}

eat (text, mustMatch) {
if (this.match(text)) {
this.index += text.length;
return true;
}

if (mustMatch) {
throw new Error(`Did not find string: ${text}`);
}

return false;
}

eatUntil (text) {
while(!this.eat(text) && this.index < this.content.length) {
this.index += 1;
}
}

allowWhitespace() {
while(whitespace.test(this.content[this.index])) {
this.index += 1;
}
}

findTags(onTag) {
while(this.index < this.content.length) {
if (this.match(')) {
if (this.match(`). The exclamation point is also valid at the beginning of a js expression, so there is no way to know if this is an expression or a comment. Because of that, we will not support spacebar comments. Instead, you can create an inline js comment (``) or use an html comment (`<!-- this is an html comment -->`).

All together,
`
findTemplateTags` would look like:

`
``js
import TagScanner from './scanner';

class Compiler {
// ... rest of class

findTemplateTags(ast) {
const result = [];

function walkChildren(children, handler) {
children.forEach(child => {
handler(child);
if (child.childNodes) {
walkChildren(child.childNodes, handler);
}
});
}

walkChildren(ast.childNodes, node => {
function onTag(scanner) {
const tagStart = scanner.index;
let allowJsExpression = true;
scanner.allowWhitespace();

if (
scanner.eat('>') ||
scanner.eat('/')
) {
// Inclusion tag or closing tag, which we don't support js in
allowJsExpression = false;
}

if (scanner.eat('else')) {
allowJsExpression = false;
}

if (scanner.eat('#')) {
scanner.allowWhitespace();

allowJsExpression = scanner.eat('if') ||
scanner.eat('unless') ||
scanner.eat('each');
}

let expression;
if (allowJsExpression) {
expression = acorn.parseExpressionAt(
scanner.content,
scanner.index,
{
allowImportExportEverywhere: true,
allowReturnOutsideFunction: true,
sourceType: 'module',
ecmaVersion: 'latest',
}
);
scanner.index = expression.end;
}

scanner.eatUntil('}}');

if (scanner.match('}')) {
// Was a triple bracket
scanner.eat('}')
}

result.push({
node,
stringPosition: scanner.contentContext,
tagStart,
tagEnd: scanner.index,
expression,
});
}

if (node.nodeName === '#text') {
new TagScanner(node.value, { type: 'nodeValue' }).findTags(onTag);
}
if (node.attrs) {
node.attrs.forEach(attr => {
if (attr.name.includes('{')) {
new TagScanner(attr.name, { type: 'attrName', attr}).findTags(onTag);
} else if (typeof attr.value === 'string') {
new TagScanner(attr.value, { type: 'attrValue', attr}).findTags(onTag);
}
});
}
});

return result;
}
}

Now that we've found all of the template tags and any js expressions, we need to move the expressions out of the html.

We will add another method to our class, prepareTemplateTags. This method will made any changes to the template tags necessary for them to be compatible with spacebars.

In this method, we will remove the expressions, and replace it with a call to helpers we will define later. There is one challenge: the expression might use variables defined in the file or imported, globals such as console or Meteor, or the template's context. At build time, we might not know which one is needed. To make it more clear, we will require accessing the current context with this. If you want the count value from the context, you would use this.count instead of count. this is valid in both a helper and in spacebars.

The next difficulty is knowing if a function call is for a helper or a function imported or defined within the script tag. Once again there is no way to know during build time if a helper exists, so we will prefer any functions in the script tag or globals. If none exist, we will fall back to using a helper.

If we write this in a file:

Count doubled is .
My user id is

We will convert it to:

Template.<template name>.helpers({
'flame_39428'() {
return this.count * 2;
},

'flame_984372'() {
return Meteor.userId();
}

'flame_329048'() {
let docsUrl = Template.instance().view.lookup('docsUrl');

return encodeURIComponent(docsUrl(this.article))
}
});
Count doubled is .
My user id is

We will handle adding the new helpers later. For now, we need to update the template tags. We will do this in a new method named prepareTemplateTags.

class Compiler {
prepareTemplateTags(templateTags) {
templateTags.slice().reverse().forEach(tag => {
if (!tag.expression) {
return;
}
});
}
}

Earlier we marked the index the template tags start and stop. We will now be modifying the strings containing the tags. To avoid the indexes becoming out of date until we are finished with a template tag, we are reversing the array. This way we start at the template tags in the end of the strings and work toward the beginning.

We will also create a name for the helper we will add:

tag.helperName = `flame_${Math.floor(Math.random() * 100000)}`;

We should keep track of which helpers we have already defined to avoid duplicates, but I am going to skip that here.

Next we need to replace the js expression with a call to the helper. The string containing the expression could be in a text node, an attribute name, or an attribute value, so we will use the stringPosition object we created when finding the template to locate the string. Later we will need to modify other parts of the template tag, so we will create a helper to handle finding the string and modifying its content:

function spliceContent(templateTag, startIndex, toRemove, replacement) {
const replace = (string) => {
const startText = string.slice(0, startIndex);
const endText = string.slice(startIndex + toRemove);

return startText + replacement + endText;
}

switch (templateTag.stringPosition.type) {
case 'nodeValue':
templateTag.node.value = replace(templateTag.node.value);
break;
case 'attrName':
templateTag.stringPosition.attr.name = replace(templateTag.stringPosition.attr.name);
break;
case 'attrValue':
templateTag.stringPosition.attr.value = replace(templateTag.stringPosition.attr.value);
}
}

The AST for the expression gives us the index in the string the expression starts and stops. We can use that when calling spliceContent:

tag.helperName = `flame_${Math.floor(Math.random() * 100000)}`;
spliceContent(
tag,
tag.expression.start,
tag.expression.end - tag.expression.start,
tag.helperName
);

The final code for prepareTemplateTags is:

  prepareTemplateTags(templateTags) {
function spliceContent(templateTag, startIndex, toRemove, replacement) {
const replace = (string) => {
const startText = string.slice(0, startIndex);
const endText = string.slice(startIndex + toRemove);

return startText + replacement + endText;
}

switch (templateTag.stringPosition.type) {
case 'nodeValue':
templateTag.node.value = replace(templateTag.node.value);
break;
case 'attrName':
templateTag.stringPosition.attr.name = replace(templateTag.stringPosition.attr.name);
break;
case 'attrValue':
templateTag.stringPosition.attr.value = replace(templateTag.stringPosition.attr.value);
}
}

templateTags.slice().reverse().forEach(tag => {
if (!tag.expression) {
return;
}

tag.helperName = `flame_${Math.floor(Math.random() * 100000)}`;
spliceContent(
tag,
tag.expression.start,
tag.expression.end - tag.expression.start,
tag.helperName
);
});
}

Next we need to add the helpers to the js. We could use strings to create the code, but we would have to make sure we properly escape and wrap method names and literals. The other option is to use AST builders. These are functions we can call to create AST nodes. We can use builders provided by reify.

To understand what AST nodes we need, we can use https://astexplorer.net/ to enter in the code we want to build and look at the resulting AST. You can also enable Transform to try the builders. It provides good autocomplete, parameter hints, and shows the output which makes it much faster to iterate. The jscodeshift transformer uses the same builders as reify.

This is the code we want to build:

Template.counter.helpers({
'flame_329048'() {
let docsUrl = Template.instance().view.lookup('docsUrl');

return encodeURIComponent(docsUrl(this.article))
}
});

We will add a new addHelpers method to our class to build these:

  addHelpers(templateTags, scriptAst, templateName) {
const helpers = [];
const {
scope
} = periscopic.analyze(scriptAst);

// AST for Template.instance().view.lookup
const lookUpFunc = b.memberExpression(
b.memberExpression(
b.callExpression(
b.memberExpression(
b.identifier('Template'),
b.identifier("instance")
),
[]
),
b.identifier("view")),
b.identifier("lookup")
);

templateTags.forEach(tag => {
if (!tag.expression || !tag.helperName) {
return;
}

// The expression shouldn't have any variable declarations
// so any variables it references are considered globals by periscopic
const {
globals
} = periscopic.analyze(tag.expression);

// If the variable referenced by the expression is defined
// within the scrip tag or is a global, we don't need to do
// anything. Otherwise, we must assume it is a template helper.
const helperLookups = Array.from(globals.keys()).filter(key => {
return !scope.has(key) && !(key in browserGlobals);
}).map(name => {
// AST for let <name> = Template.instance().view.lookup(<name>);
return b.variableDeclaration("let", [
b.variableDeclarator(
b.identifier(name),
b.callExpression(
lookUpFunc,
[b.literal(name)])
)
]);
});

const helperMethod = b.objectMethod(
"method",
b.literal(tag.helperName),
[],
b.blockStatement([
...helperLookups,
b.returnStatement(tag.expression)
])
);
helpers.push(helperMethod);
});

const completedAst = b.expressionStatement(
b.callExpression(
b.memberExpression(b.memberExpression(b.identifier("Template"), b.identifier(templateName)), b.identifier('helpers')),
[b.objectExpression(helpers)]
)
);

return print(completedAst).code;
}
}

There is a lot of code here.

  1. It first analyses the scope of the script tag. This tells us any variables and functions that are imported or defined.
  2. Next it creates the AST for accessing Template.instance().view.lookup. We will be calling lookup in helpers to access other helpers the expression uses
  3. For each expression we analyse its scope to find globals it has. Since the expression doesn't define any variables, everything it uses is a global.
  4. We identify the globals from the expression that are likely helpers. We do this by eliminating those that are defined in the script or are globals in browser
  5. We then create variable declarations to define the helpers. This looks like let counter = Template.instance().view.lookup('counter');.
  6. We then create AST for the helpers and the call to Template.<templateName>.helpers().
  7. It converts the AST into a string and returns the result.

Now we need to wire everything together:

  1. Return ast from parseJs
  2. Return templateTags from parseTags
  3. Update processFilesForTarget to call this.addHelpers
class Compiler {
parseTags(file) {
// ...

return {
// The childNode is a text node with the contents
script: scripts[0].childNodes[0].value,
template: parse5.serialize(document),
// !! this line added
templateTags,
};
}
parseJs(js, filePath) {
// ...

return {
templateName,
// !! this line added
ast
};
},

processFilesForTarget(files) {
files.forEach((file) => {
// find the contents of the template and script
const tags = this.parseTags(file);
const {
templateName = `${file.getPackageName() || 'app'}/${file.getPathInPackage()}`,
ast
} = this.parseJs(tags.script, file.getPathInPackage());

const addedHelpers = this.addHelpers(tags.templateTags, ast, templateName);
tags.script += '\n' + addedHelpers;

const output = TemplatingTools.compileTagsWithSpacebars([{
tagName: 'template',
attribs: { name: templateName },
contents: tags.template,
sourceName: file.getPathInPackage()
}]);
const scriptContent = this.babelCompiler.processOneFileForTarget(
file, tags.script
).data;

file.addJavaScript({
data: `${output.js}\n${scriptContent}`,
path: file.getPathInPackage()
});
});
}
}

We can now test this! Update counter.blaze with this content to test various scenarios:

<script>
import { ReactiveVar } from 'meteor/reactive-var';

export const name = 'counter';

Template.counter.onCreated(function helloOnCreated() {
this.counter = new ReactiveVar(0);
});

Template.counter.helpers({
counter() {
},
});

Template.counter.events({
'click button'(event, instance) {
// increment the counter when button is clicked
instance.counter.set(instance.counter.get() + 1);
},
});

function counter () {
return Template.instance().counter.get();
}

Template.registerHelper('isOdd', number => {
return number % 2 === 1;
});
Template.counter.helpers({
test() {
return 'success';
}
})
</script>

<button>Click Me</button>

<p>You've pressed the button times.</p>



<p> is odd</p>



<p>x: , y: </p>



<p>Test: </p>

It is able to call global helpers, helpers we've defined on Template.counter, functions within the script tag, access the current context with this, and call console.log. One limitation is we can't access the parent context. The .. operator spacebars uses for that is invalid js. We will address this in a later blog post.

In the next blog post, we will look into adding event listeners in the template.

At this point the compiler.js file should look like:

const parse5 = require('parse5');
const acorn = require('acorn');
const { visit } = require('ast-types');
const os = require('os');
import TagScanner from './scanner';
import periscopic from 'periscopic';
import { types, print } from 'recast';
import { browser as browserGlobals } from 'globals';

const b = types.builders;

class Compiler {
constructor() {
this.babelCompiler = new BabelCompiler;
this.babelCompiler.setDiskCacheDirectory(os.tmpdir() + '/meteor-babel-' + Math.random() + Date.now());
}

findTemplateTags(ast) {
const result = [];

function walkChildren(children, handler) {
children.forEach(child => {
handler(child);
if (child.childNodes) {
walkChildren(child.childNodes, handler);
}
});
}

walkChildren(ast.childNodes, node => {
function onTag(scanner) {
const tagStart = scanner.index;
let allowJsExpression = true;
scanner.allowWhitespace();

if (
scanner.eat('>') ||
scanner.eat('/')
) {
// Inclusion tag or closing tag, which we don't support js in
allowJsExpression = false;
}

if (scanner.eat('else')) {
allowJsExpression = false;
}

// Make sure it supports js, and jump the
// parser to where the js expression would start
if (scanner.eat('#')) {
scanner.allowWhitespace();

allowJsExpression = scanner.eat('if') ||
scanner.eat('unless') ||
scanner.eat('each');
}

let expression;
if (allowJsExpression) {
expression = acorn.parseExpressionAt(
scanner.content,
scanner.index,
{
allowImportExportEverywhere: true,
allowReturnOutsideFunction: true,
sourceType: 'module',
ecmaVersion: 'latest',
}
);
scanner.index = expression.end;
}

scanner.eatUntil('}}');

if (scanner.match('}')) {
// Was a triple bracket
scanner.eat('}')
}

result.push({
node,
stringPosition: scanner.contentContext,
tagStart,
tagEnd: scanner.index,
expression,
});
}

if (node.nodeName === '#text') {
new TagScanner(node.value, { type: 'nodeValue' }).findTags(onTag);
}
if (node.attrs) {
node.attrs.forEach(attr => {
if (attr.name.includes('{')) {
new TagScanner(attr.name, { type: 'attrName', attr }).findTags(onTag);
} else if (typeof attr.value === 'string') {
new TagScanner(attr.value, { type: 'attrValue', attr }).findTags(onTag);
}
});
}
});

return result;
}

spliceString(string, start, toRemove, replacement = '') {
const startText = string.slice(0, start);
const endText = string.slice(start + toRemove);

return startText + replacement + endText;
}

prepareTemplateTags(templateTags) {
function spliceContent(templateTag, startIndex, toRemove, replacement) {
const replace = (string) => {
const startText = string.slice(0, startIndex);
const endText = string.slice(startIndex + toRemove);

return startText + replacement + endText;
}

switch (templateTag.stringPosition.type) {
case 'nodeValue':
templateTag.node.value = replace(templateTag.node.value);
break;
case 'attrName':
templateTag.stringPosition.attr.name = replace(templateTag.stringPosition.attr.name);
break;
case 'attrValue':
templateTag.stringPosition.attr.value = replace(templateTag.stringPosition.attr.value);
}
}

templateTags.slice().reverse().forEach(tag => {
if (!tag.expression) {
return;
}

tag.helperName = `flame_${Math.floor(Math.random() * 100000)}`;
spliceContent(
tag,
tag.expression.start,
tag.expression.end - tag.expression.start,
tag.helperName
);
});
}

parseTags(file) {
// Parse the file's content to get the AST
const document = parse5.parseFragment(file.getContentsAsString());
let scripts = [];

// document.childNodes has all of the top-level tags in the file
document.childNodes = bodyContent = document.childNodes.filter(tag => {
if (tag.nodeName === 'script') {
scripts.push(tag);
return false;
}

return true;
});

const templateTags = this.findTemplateTags(document);
this.prepareTemplateTags(templateTags);

if (scripts.length !== 1) {
throw new Error(`File must have exactly one script tag in ${file.getPathInPackage()}`);
}

return {
// The childNode is a text node with the contents
script: scripts[0].childNodes[0].value,
template: parse5.serialize(document),
templateTags,
};
}

parseJs(js, filePath) {
const ast = acorn.parse(js, {
sourceType: 'module',
ecmaVersion: 'latest',
allowImportExportEverywhere: true,
locations: true,
ranges: true
});

let templateName = null;

visit(ast, {
visitExportNamedDeclaration(path) {
path.node.declaration.declarations.forEach(declaration => {
const {
id,
init
} = declaration;

if (id.type !== 'Identifier' || id.name !== 'name') {
return;
}

if (init.type !== 'Literal') {
throw new Error(`name export must be a literal in ${filePath}`);
}

templateName = declaration.init.value;
});

return false;
}
});

return {
templateName,
ast
};
}

addHelpers(templateTags, scriptAst, templateName) {
const helpers = [];
const {
scope
} = periscopic.analyze(scriptAst);

// AST for Template.instance().view.lookup
const lookUpFunc = b.memberExpression(
b.memberExpression(
b.callExpression(
b.memberExpression(
b.identifier('Template'),
b.identifier("instance")
),
[]
),
b.identifier("view")),
b.identifier("lookup")
);

templateTags.forEach(tag => {
if (!tag.expression || !tag.helperName) {
return;
}

// The expression shouldn't have any variable declarations
// so any variables it references are considered globals by periscopic
const {
globals
} = periscopic.analyze(tag.expression);

// If the variable referenced by the expression is defined
// within the scrip tag or is a global, we don't need to do
// anything. Otherwise, we must assume it is a template helper.
const helperLookups = Array.from(globals.keys()).filter(key => {
return !scope.has(key) && !(key in browserGlobals);
}).map(name => {
// AST for let <name> = Template.instance().view.lookup(<name>);
return b.variableDeclaration("let", [
b.variableDeclarator(
b.identifier(name),
b.callExpression(
lookUpFunc,
[b.literal(name)])
)
]);
});

const helperMethod = b.objectMethod(
"method",
b.literal(tag.helperName),
[],
b.blockStatement([
...helperLookups,
b.returnStatement(tag.expression)
])
);
helpers.push(helperMethod);
});

const completedAst = b.expressionStatement(
b.callExpression(
b.memberExpression(b.memberExpression(b.identifier("Template"), b.identifier(templateName)), b.identifier('helpers')),
[b.objectExpression(helpers)]
)
);

return print(completedAst).code;
}

processFilesForTarget(files) {
files.forEach((file) => {
// find the contents of the template and script
const tags = this.parseTags(file);
const {
templateName = `${file.getPackageName() || 'app'}/${file.getPathInPackage()}`,
ast
} = this.parseJs(tags.script, file.getPathInPackage());

const addedHelpers = this.addHelpers(tags.templateTags, ast, templateName);
tags.script += '\n' + addedHelpers;

const output = TemplatingTools.compileTagsWithSpacebars([{
tagName: 'template',
attribs: { name: templateName },
contents: tags.template,
sourceName: file.getPathInPackage()
}]);
const scriptContent = this.babelCompiler.processOneFileForTarget(
file, tags.script
).data;

file.addJavaScript({
data: `${output.js}\n${scriptContent}`,
path: file.getPathInPackage()
});
});
}
}

Plugin.registerCompiler({
extensions: ['blaze'],
archMatching: 'web',
isTemplate: true
}, () => new Compiler());

Have a comment? Please Send me an email


Want More? Join my monthly newsletter for new blog posts and news about Meteor and my projects.

Have a Meteor app in production? I run Monti APM for production monitoring to help improve performance and reduce errors.