Commit e17a4d10 authored by Sven Gestegård Robertz's avatar Sven Gestegård Robertz
Browse files

merge in the sending of hierarchical typedefs from branch typedefs

parents 8735f633 e5e3b844
......@@ -57,3 +57,10 @@ doc/tech_report.bbl
doc/tech_report.log
doc/tech_report.pdf
doc/tech_report.fdb_latexmk
examples/user_types/encoded_data_c
examples/user_types/encoded_data_j
examples/user_types/example_decoder
examples/user_types/example_encoder
doc/tech_report.fls
examples/dynamic/encoded_data
examples/dynamic/gen
......@@ -1130,11 +1130,39 @@ aspect C_Encoder {
" not declared");
}
protected void Decl.C_emitEncoderTypeRegister(C_env env) {
env.println("int labcomm"+env.verStr+"_encoder_type_register_" +
env.prefix + getName() + "(");
env.indent();
env.println("struct labcomm"+env.verStr+"_encoder *e");
env.unindent();
env.println(")");
env.println("{");
env.indent();
env.println("//TODO: add error handling for dependencies");
C_emitUserTypeDeps(env, null, true);
if(!isSampleDecl() || hasDependencies()) {
env.println("return labcomm"+env.verStr+"_internal_encoder_type_register(");
env.indent();
env.println("e,");
env.println("&signature_" + env.prefix + getName() + "");
env.unindent();
env.println(");");
} else {
env.println("// the type has no dependencies, do nothing");
env.println("return 0;");
}
env.unindent();
env.println("}");
}
public void TypeDecl.C_emitEncoderRegisterHandler(C_env env) {
// do nothing for type decls
C_emitEncoderTypeRegister(env);
}
public void SampleDecl.C_emitEncoderRegisterHandler(C_env env) {
C_emitEncoderTypeRegister(env);
env.println("int labcomm"+env.verStr+"_encoder_register_" +
env.prefix + getName() + "(");
env.indent();
......@@ -1146,17 +1174,28 @@ aspect C_Encoder {
C_emitUserTypeDeps(env, null, false); //XXX HERE BE DRAGONS
//currently set to false to turn off
//outputting of code
env.println("return labcomm"+env.verStr+"_internal_encoder_register(");
env.println("int result = labcomm"+env.verStr+"_internal_encoder_register(");
env.indent();
env.println("e,");
env.println("&signature_" + env.prefix + getName() + ",");
env.println("(labcomm"+env.verStr+"_encoder_function)encode_" + env.prefix + getName());
env.unindent();
env.println(");");
env.println("if(result >= 0) {\n");
env.indent();
env.println("labcomm"+env.verStr+"_encoder_type_register_" + env.prefix + getName()+"(e);");
env.println("labcomm"+env.verStr+"_internal_encoder_type_bind(");
env.indent();
env.println("e,");
env.println("&signature_" + env.prefix + getName() + ",");
env.println( (hasDependencies() ? "1" : "0") + ");");
env.unindent();
env.println("}");
env.unindent();
env.println("return result;");
env.unindent();
env.println("}");
}
}
aspect C_EncoderIoctl {
......@@ -1206,7 +1245,7 @@ aspect C_TypeDependencies {
t.C_emitUserTypeDeps(env, t.getName(), outputCode);
if(outputCode) {
System.out.println("Decl.C_emitUserTypeDeps registering "+t.getName());
env.println("labcomm"+env.verStr+"_encoder_register_"+env.prefix + t.getName()+"(e);");
env.println("labcomm"+env.verStr+"_encoder_type_register_"+env.prefix + t.getName()+"(e);");
} else { // Just output a comment
String refpath = (via == null) ? "directly" : "indirectly via "+via;
env.println(" //Depends ("+refpath+") on "+t.getName() );
......@@ -1222,7 +1261,7 @@ aspect C_TypeDependencies {
t.C_emitUserTypeRefs(env, t.getName(), outputCode);
if(outputCode) {
env.println("labcomm"+env.verStr+"_encoder_register_"+env.prefix + t.getName()+"(e);");
env.println("labcomm"+env.verStr+"_encoder_type_register_"+env.prefix + t.getName()+"(e);");
} else { // Just output a comment
String refpath = (via == null) ? "directly" : "indirectly via "+via;
env.println(" //Is referenced ("+refpath+") by "+t.getName() );
......@@ -1245,14 +1284,15 @@ aspect C_Signature {
eq TypeDecl.C_DeclTypeString() = "LABCOMM_TYPEDEF";
public void Decl.C_emitSignature(C_env env) {
C_emitFlatSignature(env);
//
// if( false && (isReferenced() || isSampleDecl())){
// Signature signature = getSignature();
// signature.C_emitSignature(env, !isSampleDecl());
// } else {
// env.println("// not emitting signature for "+getName()+isReferenced()+isSampleDecl());
// }
if( (isReferenced() || isSampleDecl())){
Signature signature = getSignature();
signature.C_emitSignature(env, !isSampleDecl());
} else {
env.println("// not emitting signature for "+getName()+isReferenced()+isSampleDecl());
}
C_emitFlatSignature(env);
// if(env.versionHasMetaData()) {
// if(isReferenced() || isSampleDecl()){
// env.println("(int (*)(void *))labcomm"+env.verStr+"_signature_" +
......@@ -1274,6 +1314,24 @@ aspect C_Signature {
}
public void Decl.C_emitFlatSignature(C_env env) {
C_emitSizeofValue(env);
env.println("static struct labcomm"+env.verStr+"_signature " +
"signature_" + env.prefix + getName() + " = {");
env.indent();
env.println("\"" + getName() + "\",");
//env.println("sizeof_" + env.prefix + getName() + ",");
//HERE BE DRAGONS? do we need sizeof for typedefs?
env.println("NULL,");
env.println("0,");
env.println("NULL,");
env.println("0,"); // index
env.println("sizeof(signature_tree_" + env.prefix + getName() + "),");
env.println("signature_tree_" + env.prefix + getName() + "");
env.unindent();
env.println(" };");
env.println("const struct labcomm"+env.verStr+"_signature " +
"*labcomm"+env.verStr+"_signature_" + env.prefix + getName() +
" = &signature_" + env.prefix + getName() + ";");
}
public void SampleDecl.C_emitFlatSignature(C_env env){
......@@ -1304,7 +1362,9 @@ aspect C_Signature {
env.println("sizeof_" + env.prefix + getName() + ",");
env.println("sizeof(signature_bytes_" + env.prefix + getName() + "),");
env.println("signature_bytes_" + env.prefix + getName() + ",");
env.println("0");
env.println("0,"); // index
env.println("sizeof(signature_tree_" + env.prefix + getName() + "),");
env.println("signature_tree_" + env.prefix + getName() + "");
env.unindent();
env.println(" };");
env.println("const struct labcomm"+env.verStr+"_signature " +
......@@ -1319,28 +1379,29 @@ aspect C_Signature {
public abstract void SignatureLine.C_emitSignature(C_env env, boolean decl);
public void TypeRefSignatureLine.C_emitSignature(C_env env, boolean isDecl){
//env.print(getIndentString());
//env.println("LABCOMM_SIGDEF_SIGNATURE(labcomm"+env.verStr+"_signature_" + env.prefix + decl.getName() +"),");
env.print(getIndentString());
// env.println("LABCOMM_SIGDEF_SIGNATURE(labcomm"+env.verStr+"_signature_" + env.prefix + decl.getName() +"),");
env.println("LABCOMM_SIGDEF_SIGNATURE(signature_" + env.prefix + decl.getName() +"),");
}
public void DataSignatureLine.C_emitSignature(C_env env, boolean decl){
// String comment = getComment();
// if (comment != null && comment.length() > 0) {
// env.println(getIndentString() + "// " + comment);
// }
// byte[] data = getData(env.version);
// if (data != null && data.length > 0) {
// env.print(getIndentString());
// env.print("LABCOMM_SIGDEF_BYTES("+data.length+", \"");
// for (int j = 0 ; j < data.length ; j++) {
// byte d = data[j];
// //if(d>='a'&&d<='z' || d>='A'&&d<='Z'|| d>='0'&&d<='9' )
// // env.print(""+(char)d);
// //else
// env.print("\\x"+Integer.toHexString(d));
// }
// env.println("\"),");
// }
String comment = getComment();
if (comment != null && comment.length() > 0) {
env.println(getIndentString() + "// " + comment);
}
byte[] data = getData(env.version);
if (data != null && data.length > 0) {
env.print(getIndentString());
env.print("LABCOMM_SIGDEF_BYTES("+data.length+", \"");
for (int j = 0 ; j < data.length ; j++) {
byte d = data[j];
//if(d>='a'&&d<='z' || d>='A'&&d<='Z'|| d>='0'&&d<='9' )
// env.print(""+(char)d);
//else
env.print("\\x"+Integer.toHexString(d));
}
env.println("\"),");
}
}
//
//
......@@ -1357,18 +1418,18 @@ aspect C_Signature {
//
//}
public void SignatureList.C_emitSignature(C_env env, boolean decl) {
// env.println("static struct labcomm_signature_data signature_tree_" +
// env.prefix + parentDecl().getName() + "[] = {");
// env.indent();
// for (int i = 0 ; i < size() ; i++) {
// SignatureLine l = getSignatureLine(i);
// l.C_emitSignature(env, decl);
// }
//
// env.println("LABCOMM_SIGDEF_END");
// env.println("};");
// env.unindent();
// env.println();
env.println("static struct labcomm_signature_data signature_tree_" +
env.prefix + parentDecl().getName() + "[] = {");
env.indent();
for (int i = 0 ; i < size() ; i++) {
SignatureLine l = getSignatureLine(i);
l.C_emitSignature(env, decl);
}
env.println("LABCOMM_SIGDEF_END");
env.println("};");
env.unindent();
env.println();
}
......@@ -1432,9 +1493,9 @@ aspect C_Constructor {
}
public void Decl.C_emitConstructor(C_env env) {
}
// }
//XXX
public void SampleDecl.C_emitConstructor(C_env env) {
// public void SampleDecl.C_emitConstructor(C_env env) {
env.println("labcomm"+env.verStr+"_set_local_index(&signature_" +
env.prefix + getName() + ");");
}
......
......@@ -21,9 +21,9 @@ aspect FlatSignature {
getType().flatSignature(list);
}
public void SampleRefType.flatSignature(SignatureList list) {
list.addInt(LABCOMM_SAMPLE_REF, "sample");
}
// public void SampleRefType.flatSignature(SignatureList list) {
// list.addInt(LABCOMM_SAMPLE_REF, "sample");
// }
public void VoidType.flatSignature(SignatureList list) {
list.addInt(LABCOMM_STRUCT, "void");
......@@ -96,9 +96,9 @@ aspect FlatSignature {
return getType().signatureComment() + " '" + getName() +"'";
}
public String SampleRefType.signatureComment() {
return "sample";
}
// public String SampleRefType.signatureComment() {
// return "sample";
// }
public String PrimType.signatureComment() {
return getName();
......
......@@ -454,10 +454,10 @@ aspect Java_Class {
env.println();
}
//public void TypeDecl.Java_emitSignature(Java_env env) {
// Signature signature = getSignature();
// signature.Java_emitSignature(env, true);
//}
public void TypeDecl.Java_emitSignature(Java_env env) {
Signature signature = getSignature();
signature.Java_emitSignature(env, true);
}
public void Decl.Java_emitSignature(Java_env env) {
//always emit the flat signature, as it is needed
......@@ -465,10 +465,10 @@ aspect Java_Class {
//the type_ids of dependent types. Therefore, flat sigs
//are used for matching
Java_emitFlatSignature(env);
//if(isReferenced() || isSampleDecl()){
// Signature signature = getSignature();
// signature.Java_emitSignature(env, !isSampleDecl());
//}
if(isReferenced() || (isSampleDecl() && hasDependencies() )){
Signature signature = getSignature();
signature.Java_emitSignature(env, !isSampleDecl());
}
}
public void Decl.Java_emitFlatSignature(Java_env env){
......@@ -497,7 +497,8 @@ aspect Java_Class {
//XXX TODO: refactor: split into a static class ("TypeDefSingleton"?)and a (smaller) dispatcher
public void Decl.Java_emitDispatcher(Java_env env, boolean isSample) {
String genericStr = ""; //env.versionHasMetaData()?"<"+getName()+">":"";
// String genericStr = ""; //env.versionHasMetaData()?"<"+getName()+">":"";
String genericStr = "<"+getName()+">";
env.println("private static Dispatcher dispatcher = new Dispatcher();");
env.println();
env.println("public SampleDispatcher getDispatcher() {");
......@@ -544,16 +545,20 @@ aspect Java_Class {
env.println("return "+isSample+";");
env.unindent();
env.println("}");
// env.println("public boolean hasStaticSignature() {");
// env.indent();
// env.println("return "+!hasDependencies()+";");
// env.unindent();
// env.println("}");
env.println("public boolean hasDependencies() {");
env.indent();
env.println("return "+hasDependencies()+";");
env.unindent();
env.println("}");
env.println();
env.println("/** return the flat signature. */");
env.println("public byte[] getSignature() {");
env.indent();
env.println("return signature;");
if(isSample) {
env.println("return signature;");
} else {
env.println("throw new Error(\"a TYPE_DEF has no flat signature\");");
}
env.unindent();
env.println("}");
env.println();
......@@ -563,15 +568,16 @@ aspect Java_Class {
// env.unindent();
// env.println("}");
// env.println();
// env.println("public void encodeSignatureMetadata(Encoder e, int index) throws IOException{");
// env.indent();
// env.println("e.encodePacked32(Constant.TYPE_DEF);");
// env.println("e.encodePacked32(index);");
// env.println("e.encodeString(getName());");
// env.println("emitSignature(e);");
// env.unindent();
// env.println("}");
// env.println();
env.println("public void encodeTypeDef(Encoder e, int index) throws IOException{");
env.indent();
if(!isSample || hasDependencies()) {
env.println("emitSignature(e);");
} else {
env.println("// the type has no dependencies, do nothing");
}
env.unindent();
env.println("}");
env.println();
env.println("public boolean canDecodeAndHandle() {");
env.indent();
env.println("return "+isSample+";");
......
Program ::= Decl*;
//TODO: Add signatures to the abstract grammar, so that
//they can be extended and refined by more than one aspect.
//sketch:
Signature ::= SignatureList FlatSignatureList:SignatureList;
SignatureList ::= SignatureLine*;
abstract SignatureLine ::= <Indent:int> <Comment:String>;
abstract DataSignatureLine : SignatureLine;
ByteArraySignatureLine : DataSignatureLine ::= <Data:byte[]>;
IntSignatureLine : DataSignatureLine ::= <Data:int>;
StringSignatureLine : DataSignatureLine ::= <Data:String>;
TypeRefSignatureLine : SignatureLine ::= Decl;
//abstract Decl ::= Type <Name:String>;
// the signature list be defined as a non-terminal attribute:
abstract Decl ::= Type <Name:String> /Signature/;
TypeDecl : Decl;
TypeDecl : Decl;
SampleDecl : Decl;
//Signatures are in the abstract grammar, so that
//they can be extended and refined by aspects.
Signature ::= SignatureList FlatSignatureList:SignatureList;
SignatureList ::= SignatureLine*;
abstract SignatureLine ::= <Indent:int> <Comment:String>;
abstract DataSignatureLine : SignatureLine;
ByteArraySignatureLine : DataSignatureLine ::= <Data:byte[]>;
IntSignatureLine : DataSignatureLine ::= <Data:int>;
StringSignatureLine : DataSignatureLine ::= <Data:String>;
TypeRefSignatureLine : SignatureLine ::= Decl;
Field ::= Type <Name:String>;
abstract Type;
VoidType : Type;
SampleRefType : Type;
PrimType : Type ::= <Name:String> <Token:int>;
UserType : Type ::= <Name:String>;
StructType : Type ::= Field*;
ParseArrayType : Type ::= Type Dim*;
abstract ArrayType :Type ::= Type Exp*;
VariableArrayType : ArrayType;
FixedArrayType : ArrayType;
VoidType : Type;
//SampleRefType : Type;
PrimType : Type ::= <Name:String> <Token:int>;
UserType : Type ::= <Name:String>;
StructType : Type ::= Field*;
ParseArrayType : Type ::= Type Dim*;
abstract ArrayType : Type ::= Type Exp*;
VariableArrayType : ArrayType;
FixedArrayType : ArrayType;
Dim ::= Exp*;
abstract Exp;
IntegerLiteral : Exp ::= <Value:String>;
VariableSize : Exp;
VariableSize : Exp;
......@@ -68,9 +68,9 @@ aspect PrettyPrint {
out.print("void");
}
public void SampleRefType.ppPrefix(PrintStream out) {
out.print("sample");
}
// public void SampleRefType.ppPrefix(PrintStream out) {
// out.print("sample");
// }
public void PrimType.ppPrefix(PrintStream out) {
out.print(getName());
......
......@@ -158,7 +158,6 @@ aspect Signature {
}
public void TypeDecl.genSigLineForDecl(SignatureList list, boolean decl) {
//System.out.println("************ TypeDecl.genSigLine("+decl+").... for "+getName());
if(decl){
getType().genSigLineForDecl(list, decl);
}else{
......@@ -167,7 +166,6 @@ aspect Signature {
}
public void SampleDecl.genSigLineForDecl(SignatureList list, boolean decl) {
//System.out.println("************ SampleDecl.genSigLine("+decl+").... for "+getName());
getType().genSigLineForDecl(list, decl);
}
......@@ -176,25 +174,20 @@ aspect Signature {
list.addInt(0, null);
}
public void SampleRefType.genSigLineForDecl(SignatureList list, boolean decl) {
list.addInt(LABCOMM_SAMPLE_REF, "sample");
}
// public void SampleRefType.genSigLineForDecl(SignatureList list, boolean decl) {
// list.addInt(LABCOMM_SAMPLE_REF, "sample");
// }
public void PrimType.genSigLineForDecl(SignatureList list, boolean decl) {
list.addInt(getToken(), null);
}
/* For UserType, the decl parameter is ignored, as a UserType
* will always be a TypeRef
*/
public void UserType.genSigLineForDecl(SignatureList list, boolean decl) {
if(decl){
//System.out.println("************ UserType.genSigLine("+decl+").... for "+getName());
TypeDecl thet=lookupType(getName());
//System.out.println("************ thet: "+thet.getName() +":"+thet.getType());
thet.genSigLineForDecl(list, decl);
}else{
//System.out.println("************ UserType.genSigLine("+decl+").... for "+getName());
TypeDecl thet = lookupType(getName());
// System.out.println("************ thet: "+thet.getName() +":"+thet.getType());
list.addTypeRef(thet, null);
}
}
public void ArrayType.genSigLineForDecl(SignatureList list, boolean decl) {
......
......@@ -437,6 +437,11 @@ come from two independent number series. To identify which
\verb+TYPE_DECL+ a particular \verb+SAMPLE_DECL+ corresponds to, the
\verb+TYPE_BINDING+ packet is used.
For sample types that do not depend on any typedefs, no \verb+TYPE_DECL+
is sent, and the \verb+TYPE_BINDING+ binds the sample id to the special
value zero to indicate that the type definition is identical to the
flattened signature.
\subsection{Example}
The labcomm declaration
......@@ -484,6 +489,42 @@ not required to do so. However, if multiple \verb+TYPE_DECL+ packets are
sent for the same \verb+typedef+, the encoder must use the same
\verb+type_id+.
\subsection{Decoding in-band type descriptions}
In LabComm, the in-band data descriptions are equivalent to \footnote{in
the sense that they contain all information needed to recreate} the data
description source (i.e., the ``.lc-file'').
%
As the type declarations (a.k.a. \emph{signatures}) are written before
sample data on every channel, they can be used to interpret data with
an unknown (by the receiver) format.
The libraries provide functionality to subscribe to (i.e., register a
\emph{handler} for) sample and type declarations.
On the low level, the handler receives an instance of the signature
data structure corresponding to the received declaration.
For higher-level processing, the Java library provides the
\verb+ASTbuilder+ class, which builds an abstract syntax tree in
the internal representation of the LabComm compiler.
That enables the user to use the complete functionality of the
LabComm compiler, e.g. code generation, on declarations received in a
LabComm stream.
In combination with on-the-fly compilation and class-loading (or
linking) that makes it possible to dynamically create handlers for
previously unknown data types. Thereby, it enables dynamic configuration
of LabComm endpoints in static languages without the overhead of
interpreting signatures (at the one-time cost of code generation and
compilation).
\section{Ideas/Discussion}:
......@@ -498,6 +539,177 @@ Java primitive types. However, it is unlikely that the entire range is actually
way of supporting the common cases is to include run-time checks for overflow in the Java encoders
and decoders.
\section{Related work}
Two in-band self-descibing communication protocols are Apache
Avro\cite{avro} and EDN, the extensible data notation developed for
Clojure and Datomic\cite{EDN}.
EDN encodes \emph{values} as UTF-8 strings. The documentation says
``edn is a system for the conveyance of values. It is not a type system,
and has no schemas.'' That said, it is \emph{extensible} in the sense
that it has a special \emph{dispatch charachter}, \verb+#+, which can
be used to add a \emph{tag} to a value. A tag indicates a semantic
interpretation of a value, and that allows the reader to support
handlers for specific tags, enabling functionality similar to that of
labcomm.
\subsection{Apache Avro}
Apache Avro is similar to LabComm in that it has a textual language
for declaring data, a binary protocol for transmitting data, and code
generation for several languages.
Avro is a larger system, including RPC \emph{protocols}, support for
using different \emph{codecs} for data compression, and \emph{schema
resolution} to support handling schema evolution and transparent
interoperability between different versions of a schema.
\subsubsection*{Data types}
In the table, the Avro type names are listed, and matched to the
corresponding LabComm type:
\begin{tabular}{|l|c|c|}
\hline
Type & Labcomm & Avro \\
\hline Primitive types \\ \hline
int & 4 bytes & varint \\
long & 8 bytes & varint \\
float & 4 bytes & 4 bytes \\
long & 8 bytes & 8 bytes \\
string & varint + utf8[] & varint + utf8[] \\
bytes & varint + byte[] & varint + byte[]\\
\hline Complex types \\ \hline
struct/record & concat of fields & concat of fields \\
arrays & varIdx[] : elements & block[] \\
map & n/a & block[] \\
union & n/a & (varint idx) : value \\
fixed & byte[n] & the number of bytes declared in
the schema\\
\hline
\end{tabular}
where
\begin{verbatim}
block ::= (varint count) : elem[count] [*1]
count == 0 --> no more blocks
[*1] for arrays, count == 0 --> end of array
if count < 0, there are |count| elements
preceded by a varint block_size to allow
fast skipping
\end{verbatim}
In maps, keys are strings, and values according to the schema.
In unions, the index indicates the kind of value and the
value is encoded according to the schema.
Note that the Avro data type \verb+bytes+ corresponds to the
LabComm declaration \verb+byte[_]+, i.e. a varaible length byte array.
\subsubsection*{the wire protocol}
\begin{tabular}{|l|c|c|}
\hline
What & LabComm & Avro \\ \hline
Data description & Binary signature & JSON schema \\
Signature sent only once pre connection& posible & possible \\
Signature sent with each sample & possible & possible \\
Data encoding & binary & binary \\
\hline
\end{tabular}
Both avro and labcomm use varints when encoding data, similar in that
they both send a sequence of bytes containing 7 bit chunks (with the
eight bit signalling more chunks to come), but they differ in range,
endianness and signedness.
\begin{verbatim}