Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
  • anders.blomdell
  • compiler-refactoring
  • labcomm2006
  • labcomm2013
  • labcomm2014
  • master
  • pragma
  • python_sig_hash
  • typedefs
  • typeref
  • v2006.0
  • v2013.0
  • v2014.0
  • v2014.1
  • v2014.2
  • v2014.3
  • v2014.4
  • v2014.5
  • v2014.6
  • v2015.0
20 results

Target

Select target project
  • anders_blomdell/labcomm
  • klaren/labcomm
  • tommyo/labcomm
  • erikj/labcomm
  • sven/labcomm
5 results
Select Git revision
  • anders.blomdell
  • compiler-refactoring
  • labcomm2006
  • labcomm2013
  • master
  • pragma
  • typedefs
  • typeref
  • v2006.0
  • v2013.0
  • v2014.0
  • v2014.1
  • v2014.2
  • v2014.3
  • v2014.4
15 results
Show changes
Showing
with 1089 additions and 371 deletions
......@@ -77,7 +77,7 @@ aspect RAPID_CodeGen {
throw new UnsupportedOperationException();
}
public void Program.RAPID_gen(String file, String prefix, int version)
public void Specification.RAPID_gen(String file, String prefix, int version)
throws IOException
{
PrintStream ps = new PrintStream(new FileOutputStream(new File(file)));
......@@ -85,7 +85,7 @@ aspect RAPID_CodeGen {
RAPID_gen(env);
}
public void Program.RAPID_gen(RAPID_env env)
public void Specification.RAPID_gen(RAPID_env env)
{
for (int i = 0; i < getNumDecl(); i++) {
getDecl(i).RAPID_gen(env);
......@@ -126,7 +126,7 @@ aspect RAPID_CodeGen {
public void SampleDecl.RAPID_gen(RAPID_env env) {
// Add type declarations
String fullName = getType().RAPID_AddType(env, getName());
String fullName = getDataType().RAPID_AddType(env, getName());
// Add signature constants
String sig_len_name = "signature_len_" + getName();
String sig_name = "signature_" + getName();
......@@ -167,7 +167,7 @@ aspect RAPID_CodeGen {
params.add("VAR LabComm_Stream st");
params.add("VAR LabComm_Decoder_Sample s");
stmts.add("VAR " + fullName + " tmp;");
getType().RAPID_AddDecodeInstr(env, stmts, "tmp", "st");
getDataType().RAPID_AddDecodeInstr(env, stmts, "tmp", "st");
stmts.add("% s.handler % tmp;");
env.addProc("Decode_And_Handle_" + getName(), params, stmts);
......@@ -201,11 +201,11 @@ aspect RAPID_CodeGen {
params.add("VAR LabComm_Encoder_Sample s");
params.add("VAR " + fullName + " val");
stmts.add("Encode_Packed st, s.user_id;");
getType().RAPID_AddEncodeInstr(env, stmts, "val", "st");
getDataType().RAPID_AddEncodeInstr(env, stmts, "val", "st");
env.addProc("Encode_" + getName(), params, stmts);
}
public String Type.RAPID_AddType(RAPID_env env, String name) {
public String DataType.RAPID_AddType(RAPID_env env, String name) {
throw new UnsupportedOperationException("RAPID code generation does (currently) not support "+getClass().getSimpleName());
}
......@@ -214,7 +214,7 @@ aspect RAPID_CodeGen {
for (int i = 0; i < getNumField(); i++) {
Field f = getField(i);
components.add(
f.getType().RAPID_AddType(env, name + "_" + f.getName()) +
f.getDataType().RAPID_AddType(env, name + "_" + f.getName()) +
" " + f.getName() + ";");
}
String typeName = env.addRecord(name, components);
......@@ -222,7 +222,7 @@ aspect RAPID_CodeGen {
}
public String FixedArrayType.RAPID_AddType(RAPID_env env, String name) {
String typeName = getType().RAPID_AddType(env, name + "_e");
String typeName = getDataType().RAPID_AddType(env, name + "_e");
if (getNumExp() > 1) {
throw new UnsupportedOperationException("RAPID generation only (currently) supports one-dimensional arrays");
}
......@@ -251,7 +251,7 @@ aspect RAPID_CodeGen {
throw new UnsupportedOperationException("RAPID code generation does not (currently) support "+getName());
}
public void Type.RAPID_AddDecodeInstr(RAPID_env env,
public void DataType.RAPID_AddDecodeInstr(RAPID_env env,
java.util.List<String> instrs,
String var_name, String stream_name) {
throw new UnsupportedOperationException("RAPID code generation does not (currently) support "+getClass().getSimpleName());
......@@ -261,7 +261,7 @@ aspect RAPID_CodeGen {
java.util.List<String> instrs,
String var_name, String stream_name) {
for (int i = 0; i < getNumField(); i++) {
getField(i).getType().RAPID_AddDecodeInstr(env, instrs,
getField(i).getDataType().RAPID_AddDecodeInstr(env, instrs,
var_name + "." + getField(i).getName(), stream_name);
}
}
......@@ -270,7 +270,7 @@ aspect RAPID_CodeGen {
java.util.List<String> instrs,
String var_name, String stream_name) {
for (int i = 1; i <= getExp(0).RAPID_getValue(); i++) {
getType().RAPID_AddDecodeInstr(env, instrs,
getDataType().RAPID_AddDecodeInstr(env, instrs,
var_name + ".e" + i, stream_name);
}
}
......@@ -305,7 +305,7 @@ aspect RAPID_CodeGen {
}
}
public void Type.RAPID_AddEncodeInstr(RAPID_env env,
public void DataType.RAPID_AddEncodeInstr(RAPID_env env,
java.util.List<String> instrs,
String var_name, String stream_name) {
throw new UnsupportedOperationException("RAPID code generation does not (currently) support "+getClass().getSimpleName());
......@@ -315,7 +315,7 @@ aspect RAPID_CodeGen {
java.util.List<String> instrs,
String var_name, String stream_name) {
for (int i = 0; i < getNumField(); i++) {
getField(i).getType().RAPID_AddEncodeInstr(env, instrs,
getField(i).getDataType().RAPID_AddEncodeInstr(env, instrs,
var_name + "." + getField(i).getName(), stream_name);
}
}
......@@ -324,7 +324,7 @@ aspect RAPID_CodeGen {
java.util.List<String> instrs,
String var_name, String stream_name) {
for (int i = 1; i <= getExp(0).RAPID_getValue(); i++) {
getType().RAPID_AddEncodeInstr(env, instrs,
getDataType().RAPID_AddEncodeInstr(env, instrs,
var_name + ".e" + i, stream_name);
}
}
......
/* Temporary aspect with forwarding methods */
aspect Refactoring {
syn int ArrayType.getNumExp() = getDim().getNumExp();
syn Exp ArrayType.getExp(int i) = getDim().getExp(i);
syn String Decl.getName() = getTypeInstance().getName();
syn DataType Decl.getDataType() = getTypeInstance().getDataType();
syn String TypeInstance.getName() = getAnnotations().getName();
public Annotations Annotations.addName(String n) {
//XXX TODO: check if name already exists
addAnnotation(new Intention("",n.getBytes()));
return this;
}
public Field.Field(TypeInstance t) {
this(t.getDataType(), t.getAnnotations());
}
public TypeInstance.TypeInstance(DataType t, String n, Annotations a) {
this(t, a.addName(n));
}
public TypeInstance.TypeInstance(DataType t, String n) {
this(t, new Annotations().addName(n));
System.out.println("WARNING! TypeInstance(DataType, String) ignoring intention list");
}
syn Annotation TypeInstance.getAnnotation(int i) = getAnnotations().getAnnotation(i);
}
......@@ -14,10 +14,9 @@ aspect Signature {
inh Decl Signature.parentDecl();
inh Decl SignatureList.parentDecl();
syn nta Signature Decl.getSignature() {
SignatureList sl = new SignatureList();
genSigLineForDecl(sl, true);
genSigLineForDecl(sl, true, this);
SignatureList fsl = new SignatureList();
flatSignature(fsl);
Signature sig = new Signature();
......@@ -100,6 +99,29 @@ aspect Signature {
return getIntBytes(getData(), version);
}
public void SignatureList.addIntentions(Set<Intention> data, String comment) {
//addString(TypeInstance.getIntentionString(data), comment);
//create IntenionSignatureLine
IntentionSignatureLine line = new IntentionSignatureLine(indent, comment, new List());
//TODO: refactor out creation of sorted list of intentions
java.util.ArrayList<Intention> sorted = new ArrayList(data);
java.util.Collections.sort(sorted, TypeInstance.intentionComp);
for(Intention i : sorted) {
line.addIntention(i);
}
addSignatureLine(line);
}
eq IntentionSignatureLine.getData(int version) {
//String tmpString = TypeInstance.getIntentionString(getIntentions());
byte[] bs = TypeInstance.getIntentionBytes(getIntentions());
return bs;
}
public void SignatureList.addString(String data, String comment) {
addSignatureLine(new StringSignatureLine(indent, comment, data));
}
......@@ -151,79 +173,200 @@ aspect Signature {
}
public void ASTNode.genSigLineForDecl(SignatureList list, boolean decl) {
public void ASTNode.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
throw new Error(this.getClass().getName() +
".genSigLineForDecl(SignatureList list)" +
" not declared");
}
public void TypeDecl.genSigLineForDecl(SignatureList list, boolean decl) {
public String TypeInstance.getIntentionString() {
return getIntentionString(intentions());
}
public static String TypeInstance.getIntentionString(List<Intention> intentions) {
if(intentions==null) return "";
Iterator<Intention> it = intentions.iterator();
return getIntentionString(it);
}
public static String TypeInstance.getIntentionString(Set<Intention> intentions) {
if(intentions==null) return "";
Iterator<Intention> it = intentions.iterator();
return getIntentionString(it);
}
public static String TypeInstance.getIntentionString(Iterator<Intention> it) {
StringBuilder sb = new StringBuilder();
while(it.hasNext()) {
Intention i = it.next();
sb.append(i.toString());
}
return sb.toString();
}
syn byte[] Intention.keyBytes() = getKey().getBytes();
syn byte[] Intention.valBytes() = getValue();
syn byte[] Intention.toByteArray() {
byte[] k = keyBytes();
byte[] v = valBytes();
int klen = Utilities.size_packed32(k.length);
int vlen = Utilities.size_packed32(v.length);
int tlen = k.length + v.length + Utilities.size_packed32(klen) + Utilities.size_packed32(vlen);
//int size = Utilities.size_packed32(tlen)+tlen;
byte result[] = new byte[tlen];
int pos=0;
// pos = Utilities.encodePacked32(tlen, result, pos, Utilities.size_packed32(tlen));
pos = Utilities.encodePacked32(k.length, result, pos, klen);
for(byte kb : k) {
result[pos++] = kb;
}
pos = Utilities.encodePacked32(v.length, result, pos, vlen);
for(byte vb : v) {
result[pos++] = vb;
}
return result;
}
public byte[] TypeInstance.getIntentionBytes() {
return getIntentionBytes(intentions());
}
public static byte[] TypeInstance.getIntentionBytes(List<Intention> intentions) {
if(intentions==null) return new byte[0];
Iterator<Intention> it = intentions.iterator();
return getIntentionBytes(it);
}
public static byte[] TypeInstance.getIntentionBytes(Set<Intention> intentions) {
if(intentions==null) return new byte[0];
Iterator<Intention> it = intentions.iterator();
return getIntentionBytes(it);
}
public static byte[] TypeInstance.getIntentionBytes(Iterator<Intention> it) {
java.util.ArrayList<byte[]> tmp = new java.util.ArrayList<byte[]>();
int tmpLen=0;
int numIntentions=0;
while(it.hasNext()) {
Intention i = it.next();
byte[] bs = i.toByteArray();
tmp.add(bs);
tmpLen+=bs.length;
numIntentions++;
}
byte result[] = new byte[tmpLen + Utilities.size_packed32(numIntentions)];
int pos = 0;
pos = Utilities.encodePacked32(numIntentions, result, 0, Utilities.size_packed32(numIntentions));
for(byte[] bs : tmp) {
for(byte b : bs) {
result[pos++] = b;
}
}
return result;
}
syn Set<Intention> Specification.emptyIntentions() = new HashSet<Intention>();
inh Set<Intention> ASTNode.noIntentions();
eq Specification.getChild(int i).noIntentions() = emptyIntentions();
syn Set<Intention> ASTNode.intentions();
eq ASTNode.intentions() = noIntentions();
eq TypeInstance.intentions() = intentionSet();
public void TypeInstance.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
// debugAnnotations(this.getName());
// list.addString(inst.getIntentionString(), "intention string");
if(addIntentions()) {
list.addIntentions(intentionSet(), "intentions");
}
getDataType().genSigLineForDecl(list, decl, this);
}
public void TypeDecl.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
//TODO intent
if(decl){
getType().genSigLineForDecl(list, decl);
getTypeInstance().genSigLineForDecl(list, decl, this);
}else{
list.addTypeRef(this, "//TODO (from list.addTypeRef)");
}
}
public void SampleDecl.genSigLineForDecl(SignatureList list, boolean decl) {
getType().genSigLineForDecl(list, decl);
public void SampleDecl.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
//TODO intent
getTypeInstance().genSigLineForDecl(list, decl, this);
}
public void VoidType.genSigLineForDecl(SignatureList list, boolean decl) {
public void VoidType.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
list.addInt(LABCOMM_STRUCT, "void");
list.addInt(0, null);
}
public void SampleRefType.genSigLineForDecl(SignatureList list, boolean decl) {
list.addInt(LABCOMM_SAMPLE_REF, "sample");
}
public void PrimType.genSigLineForDecl(SignatureList list, boolean decl) {
// public void SampleRefType.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
// list.addInt(LABCOMM_SAMPLE_REF, "sample");
// }
public void PrimType.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
list.addInt(getToken(), null);
}
/* For UserType, the decl parameter is ignored, as a UserType
* will always be a TypeRef
*/
public void UserType.genSigLineForDecl(SignatureList list, boolean decl) {
public void UserType.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
TypeDecl thet = lookupType(getName());
list.addTypeRef(thet, null);
}
public void ArrayType.genSigLineForDecl(SignatureList list, boolean decl) {
public void ArrayType.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
list.addInt(LABCOMM_ARRAY, signatureComment());
list.indent();
list.addInt(getNumExp(), null);
for (int i = 0 ; i < getNumExp() ; i++) {
getExp(i).genSigLineForDecl(list, false);
getExp(i).genSigLineForDecl(list, false, null);
}
getType().genSigLineForDecl(list, false);
getDataType().genSigLineForDecl(list, false, null);
list.unindent();
list.add(null, "}");
}
public void StructType.genSigLineForDecl(SignatureList list, boolean decl) {
public void StructType.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
list.addInt(LABCOMM_STRUCT, "struct { " + getNumField() + " fields");
list.indent();
list.addInt(getNumField(), null);
for (int i = 0 ; i < getNumField() ; i++) {
getField(i).genSigLineForDecl(list, false);
getField(i).genSigLineForDecl(list, false, inst);
}
list.unindent();
list.add(null, "}");
}
public void Field.genSigLineForDecl(SignatureList list, boolean decl) {
list.addString(getName(), signatureComment());
getType().genSigLineForDecl(list, decl);
}
// public void Field.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
// //XXX make intention
// list.addString(getName(), signatureComment());
// super.genSigLineForDecl(list, decl, inst);
// //TODOintent
// //getDataType().genSigLineForDecl(list, decl, inst);
// }
public void IntegerLiteral.genSigLineForDecl(SignatureList list, boolean decl) {
public void IntegerLiteral.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
list.addInt(Integer.parseInt(getValue()), null);
}
public void VariableSize.genSigLineForDecl(SignatureList list, boolean decl) {
public void VariableSize.genSigLineForDecl(SignatureList list, boolean decl, ASTNode inst) {
list.addInt(0, null);
}
}
......@@ -6,31 +6,49 @@ aspect TypeCheck {
// void is not allowed as a field in a struct or an array element
syn boolean Type.isNull();
eq Type.isNull() = false;
syn boolean DataType.isNull();
eq DataType.isNull() = false;
eq VoidType.isNull() = true;
eq UserType.isNull() = decl().isNull();
syn boolean TypeDecl.isNull();
eq TypeDecl.isNull() = getType().isNull();
eq TypeDecl.isNull() = getDataType().isNull();
public void ASTNode.nullTypeCheck() {}
public void Field.nullTypeCheck() {
if(getType().isNull()) {
if(getDataType().isNull()) {
error("field " + getName() + " of struct "+ declName()+ " may not be of type void");
}
}
public void ParseArrayType.nullTypeCheck() {
if(getType().isNull()) {
if(getDataType().isNull()) {
error("elements of array "+declName()+" may not be of type void");
}
}
public void ArrayType.nullTypeCheck() {
if(getType().isNull()) {
if(getDataType().isNull()) {
error("elements of array "+declName()+" may not be of type void");
}
}
}
aspect AnnotationCheck {
refine TypeCheck void ASTNode.typeCheck() {
refined(); // similar to call to super
annotationCheck();
}
public void ASTNode.annotationCheck() {}
public void TypeDecl.annotationCheck() {
Iterator<Intention> it = getTypeInstance().intentions().iterator();;
while(it.hasNext()) {
if(!it.next().getKey().equals("")) {
error("TypeDecl " + getName() + " has intentions. (Not allowed for typedefs)");
}
}
}
}
aspect User_Types {
syn String Type.getTypeName();
eq Type.getTypeName() = getClass().getName();
syn String DataType.getTypeName();
eq DataType.getTypeName() = getClass().getName();
eq PrimType.getTypeName() = getName();
eq UserType.getTypeName() = getName();
syn boolean Type.isUserType();
eq Type.isUserType() = false;
syn boolean DataType.isUserType();
eq DataType.isUserType() = false;
eq UserType.isUserType() = true;
}
......@@ -14,8 +14,8 @@ aspect Type_References {
// The dependencies on other type declarations for a Decl.
coll Set<Decl> Decl.type_dependencies() [new HashSet<Decl>()] with add;
Field contributes ((UserType)getType()).decl()
when parentDecl() != null && getType().isUserType()
Field contributes ((UserType)getDataType()).decl()
when parentDecl() != null && getDataType().isUserType()
to Decl.type_dependencies()
for parentDecl();
......@@ -24,8 +24,8 @@ aspect Type_References {
to Decl.type_dependencies()
for parentDecl();
/*
Field contributes getType().decl()
when parentDecl() != null && getType().isLeafType()
Field contributes getDataType().decl()
when parentDecl() != null && getDataType().isLeafType()
to Decl.type_dependencies()
for parentDecl();
*/
......
aspect Encoding {
public class Utilities {
/* Size of packed32 variable */
public static int size_packed32(long data)
{
long d = data & 0xffffffff;
int result = 0;
int i;
for (i = 0 ; i == 0 || d != 0; i++, d = (d >>> 7)) {
result++;
}
return result;
}
public static int encodePacked32(long value, byte[] buf, int start, int len) {
int pos = start;
byte[] tmp = new byte[5];
long v = value & 0xffffffff;
int i;
for (i = 0 ; i == 0 || v != 0 ; i++, v = (v >> 7)) {
tmp[i] = (byte)(v & 0x7f);
}
if(i != len) {
throw new Error("wrong length, was: "+i+", expected "+len);
}
for (i = i - 1 ; i >= 0 ; i--) {
buf[pos++] = (byte)(tmp[i] | (i!=0?0x80:0x00));
}
return pos;
}
}
}
aspect PrintEnv {
public abstract class PrintEnv {
protected static class Printer {
private final String indentString = " ";
private boolean newline = true; // last print ended with newline
protected PrintStream out;
private Printer printer;
/** dummy constructor motivated by the FilePrinter subclass */
protected Printer() {
this.out = null;
}
public Printer(PrintStream out) {
this.out = out;
}
public void print(PrintEnv env, String s) {
if (newline) {
newline = false;
for (int i = 0 ; i < env.getIndent() ; i++) {
out.print(indentString);
}
}
out.print(s);
}
public void println(PrintEnv env, String s) {
print(env, s);
out.println();
newline = true;
}
public void println(PrintEnv env) {
out.println();
newline = true;
}
public PrintStream getPrintStream() {
return(out);
}
public void close() throws IOException {
//do nothing
}
}
protected static class FilePrinter extends Printer {
private File file;
private IOException exception;
public FilePrinter(PrintStream out) {
super(out);
}
public FilePrinter(File f) {
file = f;
File parentFile = f.getParentFile();
if(parentFile != null) {
parentFile.mkdirs();
}
}
public void close() throws IOException {
if (out != null) {
out.close();
}
if (exception != null) {
throw exception;
}
}
public void checkOpen() {
if (out == null && exception == null) {
try {
out = new PrintStream(new FileOutputStream(file));
} catch (IOException e) {
exception = e;
}
}
}
public void print(PrintEnv env, String s) {
checkOpen();
super.print(env,s);
}
public void println(PrintEnv env, String s) {
checkOpen();
super.println(env, s);
}
}
public final int version; //labcomm version (2006 or 2014)
public final String verStr; // version suffix to append (currently _2006 and empty string)
private Printer printer;
private int indent;
private int depth;
protected PrintEnv(PrintStream out) {
this(new Printer(out));
}
protected PrintEnv(Printer printer) {
this(printer, 2014);
}
protected PrintEnv(Printer printer, int version) {
this(0, printer, version);
}
protected PrintEnv(int indent, Printer printer, int version) {
this(indent, printer, version, 0);
}
protected PrintEnv(int indent, Printer printer, int version, int depth) {
this.version = version;
this.indent = indent;
this.printer = printer;
this.verStr = LabCommVersion.versionString(version);
this.depth = depth;
}
public void close() throws IOException {
printer.close();
}
public PrintStream getPrintStream() {
return printer.getPrintStream();
}
public void indent(int amount) {
indent += amount;
}
public void indent() {
indent(1);
}
public void unindent(int amount) {
indent -= amount;
if (indent < 0) {
throw new Error("Negative indent level");
}
}
public void unindent() {
unindent(1);
}
public void print(String s) {
printer.print(this, s);
}
public void println(String s) {
printer.println(this, s);
}
public void println() {
printer.println(this, "");
}
public void incDepth() {
depth++;
}
public void decDepth() {
if(depth<=0) {
throw new RuntimeException("decDepth() called when depth = "+depth);
}
depth--;
}
public int getDepth() {
return depth;
}
public int getVersion() {
return version;
}
public int getIndent() {
return indent;
}
public Printer getPrinter() {
return printer;
}
public boolean versionHasMetaData() {
return version != 2006;
}
}
}
......@@ -5,7 +5,14 @@ aspect Version {
*/
class LabCommVersion {
public static String versionString(int version) {
return (version == 2006) ? "2006" : "";
switch(version) {
case 2006:
return "2006";
case 2014:
return "2014";
default:
throw new Error("no versionString for version "+version);
}
}
public static boolean versionHasPragma(int version) {
......
import java.util.Vector;
public class LabComm {
public static void main(String[] args) throws Exception {
String ver = null;
Vector<String> outargs = new Vector<String>();
for (String s: args) {
if (s.startsWith("--ver=")) {
String newver = s.substring(6);
if (ver != null && !ver.equals(newver)) {
throw new Exception("Mismatching versions '" + ver +
"' != '" + newver);
}
ver = newver;
} else {
outargs.add(s);
}
}
for (String s: outargs) {
System.out.println(s);
}
if (ver != null && ver.equals("2006")) {
outargs.add(0, "--ver=2006");
se.lth.control.labcomm2006.compiler.LabComm.main(outargs.toArray(
new String[0]));
} else if (ver == null || ver.equals("2014")) {
outargs.add(0, "--ver=2014");
se.lth.control.labcomm2014.compiler.LabComm.main(outargs.toArray(
new String[0]));
}
}
}
all:
.PHONY: all
all: ant-all
%:
.PHONY: test
test: ant-test
.PHONY: clean
clean: ant-clean
rm -f *~
.PHONY: distclean
distclean: clean ant-distclean
.PHONY: ant-%
ant-%:
ant $*
......@@ -126,7 +126,6 @@
<uptodate property="compile.u">
<srcfiles dir='.'>
<include name="*.java"/>
<include name="2006/*.java"/>
<include name="2014/*.java"/>
<include name="${outdir}/**/*.java"/>
</srcfiles>
......@@ -138,7 +137,7 @@
<target name="compile" depends="compile.cu" unless="compile.u">
<echo message = "compiling"/>
<javac debug="true" srcdir="." destdir="${outdir}"
includes="*.java 2006/*.java 2014/*.java gen/**/*.java"
includes="*.java 2014/*.java gen/**/*.java"
classpath="${outdir}:${tools}/beaver-rt.jar:${tools}/junit.jar"
includeantruntime="false"
fork="true" memoryMaximumSize="128M">
......@@ -170,29 +169,6 @@
</jar>
</target>
<target name="jar.all.cu">
<uptodate property="jar.all.u">
<srcfiles dir=".">
<include name="${outdir}/**/*.class"/>
</srcfiles>
<mapper type="merge"
to="labcomm_compiler.jar"/>
</uptodate>
</target>
<target name="jar.all" depends="jar.all.cu" unless="jar.all.u">
<echo message = "Generating labcomm_compiler.jar"/>
<jar destfile="labcomm_compiler.jar">
<fileset dir="${outdir}"
includes="**/*.class"/>
<zipfileset src="tools/beaver-rt.jar" includes="beaver/*.class"/>
<manifest>
<attribute name="Main-Class"
value="LabComm"/>
</manifest>
</jar>
</target>
<target name="do.version">
<!-- Wrapper that sets up package and package_path based on ${version} -->
<local name="package"/>
......@@ -218,11 +194,6 @@
</target>
<target name="all">
<antcall target="do.version">
<param name="do" value="all.version"/>
<param name="version" value="2006"/>
<param name="outdir" value="gen"/>
</antcall>
<antcall target="do.version">
<param name="do" value="all.version"/>
<param name="version" value="2014"/>
......@@ -231,19 +202,11 @@
<antcall target="compile">
<param name="outdir" value="gen"/>
</antcall>
<antcall target="do.version">
<param name="do" value="jar.version"/>
<param name="version" value="2006"/>
<param name="outdir" value="gen"/>
</antcall>
<antcall target="do.version">
<param name="do" value="jar.version"/>
<param name="version" value="2014"/>
<param name="outdir" value="gen"/>
</antcall>
<antcall target="jar.all">
<param name="outdir" value="gen"/>
</antcall>
</target>
<target name="clean">
......
#!/bin/sh
java -jar /lib/labcomm2014_compiler.jar "$@"
tech_report.aux
tech_report.bbl
tech_report.blg
tech_report.fdb_latexmk
tech_report.fls
tech_report.log
tech_report.pdf
......@@ -169,6 +169,22 @@ language covering most common use-cases.
sample string a_string;
\end{verbatim}
\subsection{The void type}
There is a type, \verb+void+, which can be used to send
a sample that contains no data.
\begin{verbatim}
typedef void an_empty_type;
sample an_empty_type no_data1;
sample void no_data2;
\end{verbatim}
\verb+void+ type can may not be used as a field in a struct or
the element type of an array.
\subsection{Arrays}
\begin{verbatim}
......@@ -198,7 +214,46 @@ only arrays of arrays.
} a_struct;
\end{verbatim}
\section{User defined types}
\subsection{Sample type refereces}
In addition to the primitive types, a sample may contain
a reference to a sample type. References are declared using
the \verb+sample+ keyword.
Examples:
\begin{verbatim}
sample sample a_ref;
sample sample ref_list[4];
sample struct {
sample ref1;
sample ref2;
int x;
int y;
} refs_and_ints;
\end{verbatim}
Sample references are need to be registered on both encoder and decoder
side, using the functions
\begin{verbatim}
int labcomm_decoder_sample_ref_register(
struct labcomm_decoder *decoder\nonumber
const struct labcomm_signature *signature);
int labcomm_encoder_sample_ref_register(
struct labcomm_encoder *encoder\nonumber
const struct labcomm_signature *signature);
\end{verbatim}
The value of an unregistered sample reference will be decoded as \verb+null+.
\subsection{User defined types}
User defined types are declared with the \verb+typedef+ reserved word,
and can then be used in type and sample declarations.
\begin{verbatim}
typedef struct {
......@@ -434,6 +489,42 @@ not required to do so. However, if multiple \verb+TYPE_DECL+ packets are
sent for the same \verb+typedef+, the encoder must use the same
\verb+type_id+.
\subsection{Decoding in-band type descriptions}
In LabComm, the in-band data descriptions are equivalent to \footnote{in
the sense that they contain all information needed to recreate} the data
description source (i.e., the ``.lc-file'').
%
As the type declarations (a.k.a. \emph{signatures}) are written before
sample data on every channel, they can be used to interpret data with
an unknown (by the receiver) format.
The libraries provide functionality to subscribe to (i.e., register a
\emph{handler} for) sample and type declarations.
On the low level, the handler receives an instance of the signature
data structure corresponding to the received declaration.
For higher-level processing, the Java library provides the
\verb+ASTbuilder+ class, which builds an abstract syntax tree in
the internal representation of the LabComm compiler.
That enables the user to use the complete functionality of the
LabComm compiler, e.g. code generation, on declarations received in a
LabComm stream.
In combination with on-the-fly compilation and class-loading (or
linking) that makes it possible to dynamically create handlers for
previously unknown data types. Thereby, it enables dynamic configuration
of LabComm endpoints in static languages without the overhead of
interpreting signatures (at the one-time cost of code generation and
compilation).
\section{Ideas/Discussion}:
......@@ -448,6 +539,234 @@ Java primitive types. However, it is unlikely that the entire range is actually
way of supporting the common cases is to include run-time checks for overflow in the Java encoders
and decoders.
\section{Related work}
Two in-band self-descibing communication protocols are Apache
Avro\cite{avro} and EDN, the extensible data notation developed for
Clojure and Datomic\cite{EDN}.
EDN encodes \emph{values} as UTF-8 strings. The documentation says
``edn is a system for the conveyance of values. It is not a type system,
and has no schemas.'' That said, it is \emph{extensible} in the sense
that it has a special \emph{dispatch charachter}, \verb+#+, which can
be used to add a \emph{tag} to a value. A tag indicates a semantic
interpretation of a value, and that allows the reader to support
handlers for specific tags, enabling functionality similar to that of
labcomm.
\subsection{Apache Avro}
Apache Avro is similar to LabComm in that it has a textual language
for declaring data, a binary protocol for transmitting data, and code
generation for several languages.
Avro is a larger system, including RPC \emph{protocols}, support for
using different \emph{codecs} for data compression, and \emph{schema
resolution} to support handling schema evolution and transparent
interoperability between different versions of a schema.
\subsubsection*{Data types}
In the table, the Avro type names are listed, and matched to the
corresponding LabComm type:
\begin{tabular}{|l|c|c|}
\hline
Type & Labcomm & Avro \\
\hline Primitive types \\ \hline
int & 4 bytes & varint \\
long & 8 bytes & varint \\
float & 4 bytes & 4 bytes \\
long & 8 bytes & 8 bytes \\
string & varint + utf8[] & varint + utf8[] \\
bytes & varint + byte[] & varint + byte[]\\
\hline Complex types \\ \hline
struct/record & concat of fields & concat of fields \\
arrays & varIdx[] : elements & block[] \\
map & n/a & block[] \\
union & n/a & (varint idx) : value \\
fixed & byte[n] & the number of bytes declared in
the schema\\
\hline
\end{tabular}
where
\begin{verbatim}
block ::= (varint count) : elem[count] [*1]
count == 0 --> no more blocks
[*1] for arrays, count == 0 --> end of array
if count < 0, there are |count| elements
preceded by a varint block_size to allow
fast skipping
\end{verbatim}
In maps, keys are strings, and values according to the schema.
In unions, the index indicates the kind of value and the
value is encoded according to the schema.
Note that the Avro data type \verb+bytes+ corresponds to the
LabComm declaration \verb+byte[_]+, i.e. a varaible length byte array.
\subsubsection*{the wire protocol}
\begin{tabular}{|l|c|c|}
\hline
What & LabComm & Avro \\ \hline
Data description & Binary signature & JSON schema \\
Signature sent only once pre connection& posible & possible \\
Signature sent with each sample & possible & possible \\
Data encoding & binary & binary \\
\hline
\end{tabular}
Both avro and labcomm use varints when encoding data, similar in that
they both send a sequence of bytes containing 7 bit chunks (with the
eight bit signalling more chunks to come), but they differ in range,
endianness and signedness.
\begin{verbatim}
LabComm Avro
unsigned 32 bit signed zig-zag coding
most significant chunk least significant chunk
first first
0 -> 00 0 -> 00
1 -> 01 -1 -> 01
2 -> 02 1 -> 02
... -2 -> 03
2 -> 04
...
127 -> 7f -64 -> 7f
128 -> 81 00 64 -> 80 01
129 -> 81 01 -65 -> 81 01
130 -> 81 02 65 -> 82 01
... ...
\end{verbatim}
\paragraph{Avro Object Container Files} can be seen as a counterpart
to a LabComm channel:
Avro includes a simple object container file format. A file has a
schema, and all objects stored in the file must be written according to
that schema, using binary encoding. Objects are stored in blocks that
may be compressed. Syncronization markers are used between blocks to
permit efficient splitting of files, and enable detection of
corrupt blocks.
The major difference is the sync markers that LabComm does not have, as
LabComm assumes that, while the transport may drop packets, there will
be no bit errors in a received packet. If data integrity is required,
that is delegated to the reader and writer for the particular transport.
\subsubsection{Representation of hierarchical data types}
For a type that contains fields of other user types, like
\begin{verbatim}
typedef struct {
int x;
int y;
} Point;
sample struct {
Point start;
Point end;
} line;
\end{verbatim}
LabComm encodes both the flattened signature and the
typedef which allows the hierarchical type structure to be
reconstructed.
%
The avro encoding is quite similar.
The \verb+Line+ example, corresponds to the two schemas
\begin{verbatim}
{"namespace": "example.avro",
"type": "record",
"name": "Point",
"fields": [
{"name": "x", "type": "int"},
{"name": "y", "type": "int"}
]
}
\end{verbatim}
and
\begin{verbatim}
{"namespace": "example.avro",
"type": "record",
"name": "Line",
"fields": [
{"name": "start", "type": "Point"},
{"name": "end", "type": "Point"}
]
}
\end{verbatim}
which is encoded in an Object Container File as
\begin{verbatim}
{"type":"record",
"name":"Line",
"namespace":"example.avro",
"fields":[{"name":"start",
"type":{"type":"record",
"name":"Point",
"fields":[{"name":"x","type":"int"},
{"name":"y","type":"int"}]}},
{"name":"end",
"type":"Point"}
]
}
\end{verbatim}
\subsubsection{Fetures not in LabComm}
Avro has a set of features with no counterpart in LabComm. They include
\paragraph{Codecs.}
Avro has multiple codecs (for compression of the data):
\begin{verbatim}
Required Codecs:
- null : The "null" codec simply passes through data uncompressed.
- deflate : The "deflate" codec writes the data block using the deflate
algorithm as specified in RFC 1951, and typically implemented using the
zlib library. Note that this format (unlike the "zlib format" in RFC
1950) does not have a checksum.
Optional Codecs
- snappy: The "snappy" codec uses Google's Snappy compression library. Each
compressed block is followed by the 4-byte, big-endian CRC32 checksum of
the uncompressed data in the block.
\end{verbatim}
\paragraph{Schema Resolution.} The main objective of LabComm is to
ensure correct operation at run-time. Therefore, a LabComm decoder
requires the signatures for each handled sample to match exactly.
Avro, on the other hand, supports the evolution of schemas and
provides support for reading data where the ordering of fields
differ (but names and types are the same), numerical types differ
but can be
\emph{promoted} (E.g., \verb+int+ can be promoted to \verb+long+,
\verb+float+, or \verb+double+.), and record fields have been added
or removed (but are nullable or have default values).
\paragraph{Schema fingerprints.} Avro defines a \emph{Parsing
Canonical Form} to define when two JSON schemas are ``the same''.
To reduce the overhead when, e.g., tagging data with the schema
there is support for creating a \emph{fingerprint} using 64/128/256
bit hashing, in combination with a centralized repository for
fingerprint/schema pairs.
\bibliography{refs}{}
\bibliographystyle{plain}
......@@ -459,21 +778,22 @@ and decoders.
\subsection{Abstract syntax}
\begin{verbatim}
Program ::= Decl*;
Specification ::= Decl*;
abstract Decl ::= Type <Name:String>;
abstract Decl ::= DataType <Name:String>;
TypeDecl : Decl;
SampleDecl : Decl;
Field ::= Type <Name:String>;
Field ::= DataType <Name:String>;
abstract Type;
VoidType : Type;
PrimType : Type ::= <Name:String> <Token:int>;
UserType : Type ::= <Name:String>;
StructType : Type ::= Field*;
ParseArrayType : Type ::= Type Dim*;
abstract ArrayType : Type ::= Type Exp*;
abstract DataType;
VoidType : DataType;
SampleRefType : DataType;
PrimType : DataType ::= <Name:String> <Token:int>;
UserType : DataType ::= <Name:String>;
StructType : DataType ::= Field*;
ParseArrayType : DataType ::= DataType Dim*;
abstract ArrayType : DataType ::= DataType Exp*;
VariableArrayType : ArrayType;
FixedArrayType : ArrayType;
......@@ -508,7 +828,7 @@ first.
The built-in data types are encoded as follows:
\begin{lstlisting}[basicstyle=\footnotesize\ttfamily]
||Type ||Encoding/Size ||
||----------||---------------------------------------------------||
||-----------||---------------------------------------------------||
||boolean || 8 bits ||
||byte || 8 bits ||
||short || 16 bits ||
......@@ -516,24 +836,27 @@ The built-in data types are encoded as follows:
||long || 64 bits ||
||float || 32 bits ||
||double || 64 bits ||
||sample_ref || 32 bits ||
||string || length (varint), followed by UTF8 encoded string ||
||array || each variable index (varint), ||
|| || followed by encoded elements ||
||struct || concatenation of encoding of each element ||
|| || in declaration order ||
\end{lstlisting}
\pagebreak
\subsection{Protocol grammar}
\label{sec:ConcreteGrammar}
\begin{lstlisting}[basicstyle=\footnotesize\ttfamily]
<packet> := <id> <length> ( <version> |
<type_decl> |
<sample_decl> |
<sample_ref> |
<type_binding> |
<sample_data> )
<version> := <string>
<sample_decl> := <sample_id> <string> <type>
<type_decl> := <type_id> <string> <type>
<sample_decl> := <sample_id> <string> <signature>
<sample_ref> := <sample_id> <string> <signature>
<type_decl> := <type_id> <string> <signature>
<type_binding> := <sample_id> <type_id>
<user_id> := 0x40..0xffffffff
<sample_id> : <user_id>
......@@ -541,10 +864,16 @@ The built-in data types are encoded as follows:
<string> := <string_length> <char>*
<string_length> := 0x00..0xffffffff
<char> := any UTF-8 char
<type> := <length> ( <basic_type> | <array_decl> | <struct_decl> | <type_id> )
<basic_type> := ( <boolean_type> | <byte_type> | <short_type> |
<signature> := <length> <type>
<type> := <length> ( <basic_type> |
<array_decl> |
<struct_decl> |
<type_id> )
<basic_type> := ( <void_type> | <boolean_type> | <byte_type> | <short_type> |
<integer_type> | <long_type> | <float_type> |
<double_type> | <string_type> )
<double_type> | <string_type> | <sample_ref_type>)
<void_type> := <struct_decl> 0 //void is encoded as empty struct
<boolean_type> := 0x20
<byte_type> := 0x21
<short_type> := 0x22
......@@ -553,13 +882,14 @@ The built-in data types are encoded as follows:
<float_type> := 0x25
<double_type> := 0x26
<string_type> := 0x27
<array_decl> := 0x10 <number_of_indices> <indices> <type>
<number_of_indices> := 0x00..0xffffffff
<sample_ref_type> := 0x28
<array_decl> := 0x10 <nbr_of_indices> <indices> <type>
<nbr_of_indices> := 0x00..0xffffffff
<indices> := ( <variable_index> | <fixed_index> )*
<variable_index> := 0x00
<fixed_index> := 0x01..0xffffffff
<struct_decl> := 0x11 <number_of_fields> <field>*
<number_of_fields> := 0x00..0xffffffff
<struct_decl> := 0x11 <nbr_of_fields> <field>*
<nbr_of_fields> := 0x00..0xffffffff
<field> := <string> <type>
<sample_data> := packed sample data sent in network order, with
primitive type elements encoded according to
......@@ -571,8 +901,9 @@ The labcomm sytem packet ids are:
\begin{lstlisting}[basicstyle=\footnotesize\ttfamily]
version: 0x01
sample_decl: 0x02
type_decl: 0x03
type_binding: 0x04
sample_ref: 0x03
type_decl: 0x04
type_binding: 0x05
\end{lstlisting}
Note that since the signature transmitted in a \verb+<sample_def>+ is
flattened, the \verb+<type>+ transmitted in a \verb+<sample_def>+ may
......
......@@ -17,7 +17,7 @@ ifeq ($(UNAME_S),Darwin)
else
cd simple ; sh compile.sh && sh run.sh
$(MAKE) -C wiki_example test
$(MAKE) -C user_types test
$(MAKE) -C user_types all
endif
$(MAKE) -C duck_typing test
$(MAKE) -C twoway test
......
gen
LABCOMM_JAR=../../compiler/labcomm_compiler.jar
LABCOMM_JAR=../../compiler/labcomm2014_compiler.jar
LABCOMM=java -jar $(LABCOMM_JAR)
all: gen/animal.py
......
#!/usr/bin/python
import labcomm
import labcomm2014
import animal
import StringIO
......@@ -10,7 +10,7 @@ class Animal:
if __name__ == '__main__':
buf = StringIO.StringIO()
encoder = labcomm.Encoder(labcomm.StreamWriter(buf))
encoder = labcomm2014.Encoder(labcomm2014.StreamWriter(buf))
encoder.add_decl(animal.cow.signature)
encoder.add_decl(animal.dog.signature)
encoder.add_decl(animal.duck.signature)
......@@ -22,12 +22,12 @@ if __name__ == '__main__':
theAnimal.says = 'Quack'
encoder.encode(theAnimal, animal.duck.signature)
buf.seek(0)
decoder = labcomm.Decoder(labcomm.StreamReader(buf))
decoder = labcomm2014.Decoder(labcomm2014.StreamReader(buf))
try:
while True:
value,decl = decoder.decode()
if value:
print decl.name, 'says', value.says
print decl.name, 'says', value
pass
pass
pass
......
all:
sh dynamic.sh
sh dynamic_type.sh
sh test.sh
sh test_type.sh
clean:
-rm test/*.class
-rm encoded_data
-rm dynamic_out
distclean:
distclean: clean
#dummy script to test the on-the-fly compilation
java -jar ../../compiler/labcomm_compiler.jar --java=gen --javapackage=gen simple.lc
java -jar ../../compiler/labcomm2014_compiler.jar --java=gen --javapackage=gen simple.lc
javac -cp .:gen:../../lib/java/labcomm.jar gen/*.java
javac -cp .:gen:../../lib/java/labcomm2014.jar gen/*.java
# compile static encoder and decoder
javac -cp .:gen:../../lib/java/labcomm.jar test/StaticEncoder.java
javac -cp .:gen:../../lib/java/labcomm.jar test/StaticDecoder.java
javac -cp .:gen:../../lib/java/labcomm2014.jar test/StaticEncoder.java
javac -cp .:gen:../../lib/java/labcomm2014.jar test/StaticDecoder.java
# compile dynamic part
javac -cp .:../../compiler/labcomm_compiler.jar:../../lib/java/labcomm.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar test/DynamicPart.java
javac -cp .:../../compiler/labcomm2014_compiler.jar:../../lib/java/labcomm2014.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar test/DynamicPart.java
javac test/HandlerContext.java
# run static encoder
java -cp .:gen:../../lib/java//labcomm.jar test.StaticEncoder encoded_data
java -cp .:gen:../../lib/java//labcomm2014.jar test.StaticEncoder encoded_data
# run dynamic part
java -cp .:../../compiler/labcomm_compiler.jar:../../lib/java/labcomm.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar test.DynamicPart simple.lc handlers2.txt encoded_data dynamic_out
java -cp .:../../compiler/labcomm2014_compiler.jar:../../lib/java/labcomm2014.jar:../../lib/tools/beaver.jar:../../lib/tools/beaver-rt.jar:../../lib/tools/jastadd2.jar:../../lib/tools/JFlex.jar:../../lib/tools/proj.jar test.DynamicPart simple.lc handlers2.txt encoded_data dynamic_out
# run static decoder
java -cp .:gen:../../lib/java//labcomm.jar test.StaticDecoder dynamic_out
java -cp .:gen:../../lib/java//labcomm2014.jar test.StaticDecoder dynamic_out