Greetings XLM Community! I hope your weekend is going well and you are all staying safe and happy.

As DevNullProd prepares it's submission for the SCF Seed Fund, we've been looking into some of the technical aspects of the XLM protocol as we flush out our MVP. A particular sticking point that we've encountered is around the External Data Representation (XDR) format that XLM uses to represent data such as transaction envelopes and results.

We haven't been able to find extensive documentation pertaining to this format, and will need to dive into it in our application. Take the following javascript code for example:

`var StellarSdk = require('stellar-sdk')
var server = new StellarSdk.Server('https://horizon.stellar.org');

var es = server.transactions()
.transaction("73ccc1c241fe314baa0280c7629b98644469faccdfdb0add654ae7e9b0dc9818")
.call()
.then(function(tx){
console.log("Envelope:")
console.log(JSON.stringify(StellarSdk.xdr.TransactionEnvelope.fromXDR(tx.envelope_xdr, 'base64'), null, " "));
})`

This results in the following:

Envelope:
{
"_switch": {
"name": "envelopeTypeTx",
"value": 2
},
"_arm": "v1",
"_value": {
"_attributes": {
"tx": {
"_attributes": {
"sourceAccount": {
"_switch": {
"name": "keyTypeEd25519",
"value": 0
},
"_arm": "ed25519",
"_armType": {
"_length": 32,
"_padding": 0
},
"_value": {
"type": "Buffer",
"data": [
153,
199,
<snip>
]
}
},
"fee": 2000,
"seqNum": {
"low": 3340403,
"high": 27330061,
"unsigned": false
},
"timeBounds": {
"_attributes": {
"minTime": {
"low": 0,
"high": 0,
"unsigned": true
},
"maxTime": {
"low": 1606681728,
"high": 0,
"unsigned": true
}
}
},
"memo": {
"_switch": {
"name": "memoNone",
"value": 0
},
"_arm": {},
"_armType": {}
},
"operations": [
{
"_attributes": {
"body": {
"_switch": {
"name": "manageBuyOffer",
"value": 12
},
"_arm": "manageBuyOfferOp",
"_value": {
"_attributes": {
"selling": {
"_switch": {
"name": "assetTypeCreditAlphanum4",
"value": 1
},
"_arm": "alphaNum4",
"_value": {
"_attributes": {
"assetCode": {
"type": "Buffer",
"data": [
85,
83,
68,
0
]
},
"issuer": {
"_switch": {
"name": "publicKeyTypeEd25519",
"value": 0
},
"_arm": "ed25519",
"_armType": {
"_length": 32,
"_padding": 0
},
"_value": {
"type": "Buffer",
"data": [
232,
166,
<snip>
]
}
}
}
}
},
"buying": {
"_switch": {
"name": "assetTypeNative",
"value": 0
},
"_arm": {},
"_armType": {}
},
"buyAmount": {
"low": 0,
"high": 0,
"unsigned": false
},
"price": {
"_attributes": {
"n": 1,
"d": 10000
}
},
"offerId": {
"low": 378761140,
"high": 0,
"unsigned": false
}
}
}
}
}
},
{
"_attributes": {
"body": {
"_switch": {
"name": "manageBuyOffer",
"value": 12
},
"_arm": "manageBuyOfferOp",
"_value": {
"_attributes": {
"selling": {
"_switch": {
"name": "assetTypeCreditAlphanum4",
"value": 1
},
"_arm": "alphaNum4",
"_value": {
"_attributes": {
"assetCode": {
"type": "Buffer",
"data": [
85,
83,
68,
0
]
},
"issuer": {
"_switch": {
"name": "publicKeyTypeEd25519",
"value": 0
},
"_arm": "ed25519",
"_armType": {
"_length": 32,
"_padding": 0
},
"_value": {
"type": "Buffer",
"data": [
232,
166,
<snip>
]
}
}
}
}
},
"buying": {
"_switch": {
"name": "assetTypeNative",
"value": 0
},
"_arm": {},
"_armType": {}
},
"buyAmount": {
"low": 674980527,
"high": 3,
"unsigned": false
},
"price": {
"_attributes": {
"n": 300455075,
"d": 1568229641
}
},
"offerId": {
"low": 0,
"high": 0,
"unsigned": false
}
}
}
}
}
}
],
"ext": {
"_switch": 0,
"_arm": {},
"_armType": {}
}
}
},
"signatures": [
{
"_attributes": {
"hint": {
"type": "Buffer",
"data": [
27,
205,
218,
206
]
},
"signature": {
"type": "Buffer",
"data": [
106,
76,
<snip>
]
}
}
}
]
}
}
}

Our question is how to interpret these various fields. We can take a guess at some things, things like _switch indicate an enumeration of one of several values, but the semantics of other things such as _arm, _armtype, etc are unclear, as well as how to best unserialize buffers into human friendly strings.

The XDR definition files help somewhat in terms of the structure of serialized objects but what would be of most use would be developer documentation on how to interpret this data and use it in an external application. Additional some code examples on how to parse and use the data would be of great assistance.

Any insights into this would be highly appreciated!

After investigating this a bit further, the format as returned by the "fromXDR" methods is in a standard format used by js-xdr. It can be converted to/from JSON using json-xdr

var toJSON = require('json-xdr').toJSON
toJSON(StellarSdk.xdr.TransactionEnvelope.fromXDR(tx.envelope_xdr, 'base64'))

Results in:

{
"_type": "envelopeTypeTx",
"v1": {
"tx": {
"sourceAccount": {
"_type": "keyTypeEd25519",
"ed25519": "mcfP1x5DZziNHLxXU1oyYLS+ujC/WNUI5MpshBvN2s4="
},
"fee": 2000,
"seqNum": "117381718196025459",
"timeBounds": {
"minTime": "0",
"maxTime": "1606681728"
},
"memo": {
"_type": "memoNone"
},
"operations": [
{
"body": {
"_type": "manageBuyOffer",
"manageBuyOfferOp": {
"selling": {
"_type": "assetTypeCreditAlphanum4",
"alphaNum4": {
"assetCode": "VVNEAA==",
"issuer": {
"_type": "publicKeyTypeEd25519",
"ed25519": "6KYahh5gr2D4B3PgY0blxyy+Wdyt2jdgjVjvQlEdn9w="
}
}
},
"buying": {
"_type": "assetTypeNative"
},
"buyAmount": "0",
"price": {
"n": 1,
"d": 10000
},
"offerId": "378761140"
}
}
},
{
"body": {
"_type": "manageBuyOffer",
"manageBuyOfferOp": {
"selling": {
"_type": "assetTypeCreditAlphanum4",
"alphaNum4": {
"assetCode": "VVNEAA==",
"issuer": {
"_type": "publicKeyTypeEd25519",
"ed25519": "6KYahh5gr2D4B3PgY0blxyy+Wdyt2jdgjVjvQlEdn9w="
}
}
},
"buying": {
"_type": "assetTypeNative"
},
"buyAmount": "13559882415",
"price": {
"n": 300455075,
"d": 1568229641
},
"offerId": "0"
}
}
}
],
"ext": {
"_type": 0
}
},
"signatures": [
{
"hint": "G83azg==",
"signature": "akwqqV7Xf6t8Kr0HTaCDM5jxfsZNhdgjkubJ84SN0zSege5sxh8gDc/E5EPbrE/Nhgf4ryl1j3yce5kmmIRZAQ=="
}
]
}
}

Hope this helps!

Thanks for the feedback @bkolobara, I will certainly checkout the XDR RFC. Using json-xdr in conjunction with with 'fromXDR' methods in the stellar-sdk allows us to convert the XDR into a higher level JSON format which we can operate on.

As a followup to the discussion above, I noticed that certain fields such as assetCode and signature hint (perhaps others) are not converted from XDR when the transaction envelope is:

<snip>
"operations": [
{
"body": {
"_type": "manageSellOffer",
"manageSellOfferOp": {
"selling": {
"_type": "assetTypeNative"
},
"buying": {
"_type": "assetTypeCreditAlphanum4",
"alphaNum4": {
"assetCode": "VEVSTg==",
"issuer": {
"_type": "publicKeyTypeEd25519",
"ed25519": "zQHV3rvLLHGsv/oPgdXAi9Tg1tSOJKW2FCyBmdkPdjM="
}
}
},
<snip>

Of course the assetCode above can be converted using the call:

StellarSdk.xdr.AssetCode4.fromXDR("VEVSTg==", 'base64')

But is there a way to automatically do this recursively so that we don't need to traverse the structure manually?

As always appreciate the help!

As a temporary (hacky) workaround for the time being, we're using jsonpath in conjunction w/ the fields we converted from xdr to json to dive into json to convert the asset codes to their string equivalents:

jsonpath.apply(tx.envelope, "$..alphaNum4.assetCode", (c) => StellarSdk.xdr.AssetCode4.fromXDR(c, 'base64').toString().replace(/\0/g, ''))
jsonpath.apply(tx.envelope, "$..alphaNum12.assetCode", (c) => StellarSdk.xdr.AssetCode12.fromXDR(c, 'base64').toString().replace(/\0/g, ''))

Hope this helps others in the same situation!

5 days later

Greetings! While continuing to parse the transactions from an XLM server, we encountered another stumbling block. Hopefully the answer to this one is quick as well!

Particularily this pertaining to parsing Ed25519 public keys are represented in the transaction data, for example as seen here:

"destination": -{
"_type": "publicKeyTypeEd25519",
"ed25519": "bG6V1yJrZZ7OWLBBaL3UYQ0ufp/jfQt49fLNsjX2b+Q="
},

The format of this key is not currently known to us and there doesn't seem to be much documentation on how to extract a public XLM address from it. Is there are simple call to the KeyPair or StrKey classes in the Javascript SDK that would yield such value?

Thank you in advance!

    DevNullProd

    Looks like a base64 encoded byte string.

    base64 decode, and then create an strkey string from the raw data.

    @dzham I don't believe it's base64 encoded. Asking in one of the XLM chat channels, I received the following solution using the python sdk:

    `
    from stellar_sdk.keypair import Keypair
    from stellar_sdk.xdr.curve25519_public import Curve25519Public

    xdr = "bG6V1yJrZZ7OWLBBaL3UYQ0ufp/jfQt49fLNsjX2b+Q="
    curve_public = Curve25519Public.from_xdr(xdr)
    keypair = Keypair.from_raw_ed25519_public_key(curve_public.key)
    print(keypair.public_key) # GBWG5FOXEJVWLHWOLCYEC2F52RQQ2LT6T7RX2C3Y6XZM3MRV6ZX6IJHP
    `

    Looking at the libraries themselves, the only function from your example that I haven't been able to find the equivalent of is: from_raw_ed25519_public_key

    Static Keypair functions such as fromRawEd25519Seed and fromPublicKey are present in both but not fromRawEd25519PublicKey. Does anyone know if this is an omission or if it available via another call?

    Many thanks!

    For the record the follow does not produce the correct result:

    curve = ss.xdr.Curve25519Public.fromXDR("bG6V1yJrZZ7OWLBBaL3UYQ0ufp/jfQt4")
    kp = new ss.Keypair({type : 'ed25519', publicKey: curve.key()})
    kp.publicKey() => GBREONSWGF4UU4S2LI3U6V2MIJBGCTBTKVMVCMDVMZYC62TGKF2DIXGP
    
    sk = ss.StrKey.decodeEd25519PublicKey(kp.publicKey())
    kp = new ss.Keypair({type : 'ed25519', publicKey: sk})
    kp.publicKey() =>  GBREONSWGF4UU4S2LI3U6V2MIJBGCTBTKVMVCMDVMZYC62TGKF2DIXGP

    Big shout out to Overcat who helped resolve this issue via github. The solution:

    const StellarBase = require("stellar-base")
    
    const xdr = "bG6V1yJrZZ7OWLBBaL3UYQ0ufp/jfQt49fLNsjX2b+Q="
    const keypair = new StellarBase.Keypair({type: "ed25519", publicKey: Buffer.from(xdr, 'base64')})
    console.log("Public Key: ", keypair.publicKey())