How to work with schemaless key-value storage using Scala macros.

1. H E R D I NG { T Y P E S }
WITH SCALA #MACROS
Marina Sigaeva

2. ROADMAP

3. ROADMAP
▸ How Java driver stores
data in Aerospike

4. ROADMAP
▸ How Java driver stores
data in Aerospike
▸ What driver has to do

5. ROADMAP
▸ How Java driver stores
data in Aerospike
▸ What driver has to do
▸ The type safe solution
with Scala macros

6. KEY META
BINS
K1 (id,“Bob”)
K2 Bin1, Bin2, .. BinN
PEOPLE
NS1
SET:

7. KEY META
BINS
K1 (id,“Joe”)
K2 Bin1, Bin2, .. BinN
NS1
SET: PEOPLE

8. KEY META
BINS
K1 (id,1234)
K2 Bin1, Bin2, .. BinN
NS1
SET: PEOPLE

9. DS

10. DS DS

11. DS DS DS

12. Java
object DBConnector {
val config = ConfigFactory.load()
val hosts = List(config.getString("aerospike.host"))
val port = config.getInt("aerospike.port")
val namespace = config.getString("aerospike.namespace")
val setName = config.getString("aerospike.setName")
val database: AsyncClient = new AsyncClient(new AsyncClientPolicy, hosts.map(new Host(_, port)): _*)
val key1 = new Key("namespace", "setName", new StringValue("key1"))
val key2 = new Key("namespace", "setName", new StringValue("key2"))
val key3 = new Key("namespace", "setName", new StringValue("key3"))
database.put(new WritePolicy, key1, Seq(new Bin("bin1", new StringValue("abcd"))): _*)
database.put(new WritePolicy, key2, Seq(new Bin("bin2", new IntegerValue(2))): _*)
database.put(new WritePolicy, key3, Seq(new Bin("bin3", new BooleanValue(true))): _*)
val record1 = database.get(new BatchPolicy, key1)
val record2 = database.get(new BatchPolicy, key2)
val record3 = database.get(new BatchPolicy, key3)
val res1 = longParsingFunction(record1)
val res2 = longParsingFunction(record2)
val res3 = longParsingFunction(record3)
def longParsingFunction[T](record: Record): T = {
val outValue: Map[String, Option[$tpe]] = {
val jMap = record.bins.view collect {
case (name, bt: Any) =>
val res = fetch(bt)
if (res.isEmpty && r.bins.nonEmpty) throw new Exception("Wrong type!")
else name -> res

13. Scalaobject DBConnector {
val db = new Database()
db.put("key1", "bin1", "abcd")
db.put("key2", "bin2", 2)
db.put("key3", "bin3", true)
val res1 = db.get[String]("key1")
val res2 = db.get[Int]("key2")
val res3 = db.get[Boolean]("key3")
}

14. Part 1
data storing

15. package ru.tinkoff.example
object DBConnector {
…
database.put(new WritePolicy,
new Key("namespace", "setName", new StringValue("key1")),
Seq(new Bin("binName1", new StringValue("binValue"))):_*)
database.put(new WritePolicy,
new Key("namespace", "setName", new StringValue("key2")),
Seq(new Bin("binName2", new IntegerValue(2))):_*)
database.put(new WritePolicy,
new Key("namespace", "setName", new StringValue("key3")),
Seq(new Bin("binName3", new BooleanValue(true))):_*)
}
7 Key types
12 Bin types

16. db.put("key1", "bin1", "abcd")
db.put("key2", "bin2", 8)
db.put("key3", "bin3", true)
db.put("key4", "bin4", List(1.8, 33.8, 128))
db.put("key4", "bin4", List(true, false))

17. db.put("key5", "cat", Cat("Rex", 2))
db.put("key6", "trucks",
List(Truck("F1", 8), Truck("F2", 23),
Truck("F3", 11)))
db.put("key7", "students",
List(Student("Jack Green", A),
Student("Sara Lee", A),
Student("Cameron Roonie", B)))
db.put("key8", "hList”",
"qwerty" :: 2 :: false :: HNil)

18. db.put("key5", "cat", Cat("Rex", 2))
db.put("key6", "trucks",
List(Truck("F1", 8), Truck("F2", 23),
Truck("F3", 11)))
db.put("key7", "students",
List(Student("Jack Green", A),
Student("Sara Lee", A),
Student("Cameron Roonie", B)))
db.put("key8", "hList”",
"qwerty" :: 2 :: false :: HNil)
UNLIMITED

19. Boolean
Short
ByteInt
Long
H
List
Caseclass
Map

20. val hList = 123 :: "abcdef" :: true :: HNil
+———————————————————————-+
| hList |
+———————————————————————-+
| MAP('{"0":123, "1":"abcdef", "2":1}') |
+———————————————————————-+
val cat0 = Cat("Lewis", 3)
+—————————————————————-—-+
| cat0 |
+———————————————————————+
| MAP('{"name":"Lewis", "age":3}') |
+———————————————————————+

21. Part 2
getting data

22. get(policy: BatchPolicy, listener: BatchSequenceListener, records: util.List[Ba
get(policy: BatchPolicy, listener: RecordSequenceListener, keys: Array[Key], b
get(policy: BatchPolicy, listener: RecordArrayListener, keys: Array[Key], binN
get(policy: BatchPolicy, listener: RecordSequenceListener, keys: Array[Key])
get(policy: BatchPolicy, listener: RecordArrayListener, keys: Array[Key])
get(policy: Policy, listener: RecordListener, key: Key, binNames: String *)
get(policy: Policy, listener: RecordListener, key: Key)
get(policy: BatchPolicy, listener: BatchListListener, records: util.List[BatchRe
get(policy: BatchPolicy, records: util.List[BatchRead])
get(policy: BatchPolicy, keys: Array[Key], binNames: String *)
get(policy: BatchPolicy, keys: Array[Key])
get(policy: Policy, key: Key, binNames: String *)
get(policy: Policy, key: Key)
m
m
m
m
m
m
m
m
m
m
m
m
m

23. 13
13
methods

24. public final class Record {
//Map of requested name/value bins.
public final Map<String,Object> bins;
public final int generation;
public final int expiration;
…
}

25. val results = database.get(new BatchPolicy(),
new RecordArrayListener(),
List("Lewis", "Mona", "Rex")
.map(key => createKey(key)))
.map(record =>
longParsingCatFunction(record))
def longParsingCatFunction[T](
record: Record): Option[T] = {
…
}

26. db.get[List[Cat]](List("Lewis", "Mona", "Rex"))

27. WHAT DOES THE DRIVER
HAVE TO DO?

28. WHAT DOES THE DRIVER
HAVE TO DO?
▸ More concise API

29. WHAT DOES THE DRIVER
HAVE TO DO?
▸ More concise API
▸ Take care of serialization

30. WHAT DOES THE DRIVER
HAVE TO DO?
▸ More concise API
▸ Take care of serialization
▸ To work out of the box

31. Part 3
implementation

32. def call[K, B](action: Action, key: K, bin: B)
(implicit kw: KeyWrapper[K],
bw: BinWrapper[B],
pw: Option[WritePolicy] = None) = ???

33. def call[K, B](action: Action, key: K, bin: B)
(implicit kw: KeyWrapper[K],
bw: BinWrapper[B],
pw: Option[WritePolicy] = None) = ???
call(Put, “key”, “abcd”)

34. def call[K, B](action: Action, key: K, bin: B)
(implicit kw: KeyWrapper[K],
bw: BinWrapper[B],
pw: Option[WritePolicy] = None) = ???
call(Put, “key”, “abcd”)(
new KeyWrapper[String]{},
new BinWrapper[String]{}, None)

35. def call[K, B](action: Action, key: K, bin: B)
(implicit kw: KeyWrapper[K],
bw: BinWrapper[B],
pw: Option[WritePolicy] = None) = ???
def put[K, B](key: K, bin: B)(call(Put, key, bin))
def get[K, B](key: K)(call(Get, key))

36. trait KeyWrapper[K] {
def apply(k: K): Key = new Key(toValue(k))
def toValue(v: K): Value = v match {
case s: String => new StringValue(s)
case h: HList => new MapValue(toMap(h))
…
}
}

37. new KeyWrapper[String] {
def apply(k: String): Key = ???
}
new KeyWrapper[HList] {
def apply(k: HList): Key = ???
}
new KeyWrapper[ ] {
def apply(k: ): Key = ???
}
new KeyWrapper[ ] {
def apply(k: ): Key = ???
}

38. Macros
Context
Symbol Type Tree

39. import c.universe._
Apply(
Select(Literal(Constant(1)), TermName("$plus")),
List(Literal(Constant(1)))
)
1 + 1

40. import c.universe._
Apply(
Select(Literal(Constant(1)), TermName("$plus")),
List(Literal(Constant(1)))
)
1 + 1
q"1 + 1"

41. object KeyWrapper {
implicit def materialize[K]: KeyWrapper[K] = macro implKey[K]
def implKey[K: c.WeakTypeTag](c: Context): c.Expr[KeyWrapper[K]] = {
import c.universe._
val tpe = weakTypeOf[K]
val imports = q"""
import com.aerospike.client.{Key, Value} """
c.Expr[KeyWrapper[K]] {
q"""
$imports
new KeyWrapper[$tpe] {}
"""
}
}
}

42. object KeyWrapper {
implicit def materialize[K]: KeyWrapper[K] = macro implKey[K]
def implKey[K: c.WeakTypeTag](c: Context): c.Expr[KeyWrapper[K]] = {
import c.universe._
val tpe = weakTypeOf[K]
val imports = q"""
import com.aerospike.client.{Key, Value} """
c.Expr[KeyWrapper[K]] {
q"""
$imports
new KeyWrapper[$tpe] {}
"""
}
}
}

43. object KeyWrapper {
implicit def materialize[K]: KeyWrapper[K] = macro implKey[K]
def implKey[K: c.WeakTypeTag](c: Context): c.Expr[KeyWrapper[K]] = {
import c.universe._
val tpe = weakTypeOf[K]
val imports = q"""
import com.aerospike.client.{Key, Value} """
c.Expr[KeyWrapper[K]] {
q"""
$imports
new KeyWrapper[$tpe] {}
"""
}
}
}

44. object Usage {
insertData(materialize[String])
}

45. object Usage {
insertData({
import com.aerospike.client.{Key, Value};
{
final class $anon extends KeyWrapper[String] {
def <init>() = {
super.<init>();
()
};
<empty>
};
new $anon()
}
})
}

46. new KeyWrapper[String]
new KeyWrapper[Int]
new KeyWrapper[HList]
new KeyWrapper[Cat]

47. trait BinWrapper[B] {
def apply(v: B): Bin = new Bin("name", toValue(v))
def toValue(v: B): Value = v match {
case s: String => new StringValue(s)
case h: HList => new MapValue(toMap(h))
case _ => throw new Exception("Wrong type")
}
def apply(r: Record): Map[String, B] =
r.bins.collect {
case (name, something) =>
name -> fetch(something)
}.toMap
def fetch(donkey: Any): B
}

48. trait BinWrapper[B] {
def apply(v: B): Bin = new Bin("name", toValue(v))
def toValue(v: B): Value = v match {
case s: String => new StringValue(s)
case h: HList => new MapValue(toMap(h))
case _ => throw new Exception("Wrong type")
}
def apply(r: Record): Map[String, B] =
r.bins.collect {
case (name, something) =>
name -> fetch(something)
}.toMap
def fetch(donkey: Any): B
}

49. object BinWrapper {
implicit def materialize[T]: BinWrapper[T] = macro matBin[T]
def matBin[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[BinWrapper[T]] = {
import c.universe._
val tpe = weakTypeOf[T]
val imports = q""" import …"""
val fetchValue = ???
c.Expr[BinWrapper[T]] {
q""" $imports
new BinWrapper[$tpe] {
$fetchValue
}"""
}
}
}

50. object BinWrapper {
implicit def materialize[T]: BinWrapper[T] = macro matBin[T]
def matBin[T: c.WeakTypeTag](c: blackbox.Context): c.Expr[BinWrapper[T]] = {
import c.universe._
val tpe = weakTypeOf[T]
val imports = q""" import …"""
val fetchValue = ???
c.Expr[BinWrapper[T]] {
q""" $imports
new BinWrapper[$tpe] {
$fetchValue
}"""
}
}
}

51. val tpe = weakTypeOf[T]
val fetchValue = tpe match {
case t if t =:= weakTypeOf[String] =>
q""" def fetch(any: Any): $tpe = any match {
case v: String => v
case _ => throw new Exception("Wrong type")
} """
case t if isHList(t) =>
q""" def fetch(any: Any): $tpe = any match {
case mv: MapValue => mv.getObject match {
case m: Map[String, Any] =>
parse(m).toHList[$tpe].getOrElse{}
case _ => throw new Exception("Wrong type")
}
case _ => throw new Exception("Wrong type")
} """
...
}

52. Expr[ru.tinkoff.BinWrapper[shapeless.::[Int,shapeless.::[Boolean,shapeless.HNil]]]]({
import com.aerospike.client.{Key, Value};
import collection.JavaConversions._;
import com.aerospike.client.Value._;
import shapeless._;
import syntax.std.traversable._;
{
final class $anon extends BinWrapper[shapeless.::[Int,shapeless.::[Boolean,shapeless.HNil]]] {
def <init>() = {
super.<init>();
()
};
def fetch(any: Any): shapeless.::[Int,shapeless.::[Boolean,shapeless.HNil]] = any match {
case (mv @ (_: MapValue)) => mv.getObject match {
case (m @ (_: Map[String, Any])) =>
parse(m).toHList[shapeless.::[Int,shapeless.::[Boolean,shapeless.HNil]]].getOrElse{}
case _ => throw new Exception("Wrong type")
}
case _ => throw new Exception("Wrong type")
}
};
new $anon()
}
})
BinWrapper.matBin[Int::Boolean::HNil]

53. Expr[ru.tinkoff.BinWrapper[shapeless.::[Int,shapeless.::[Boolean,shapeless.HNil]]]]({
import com.aerospike.client.{Key, Value};
import collection.JavaConversions._;
import com.aerospike.client.Value._;
import shapeless._;
import syntax.std.traversable._;
{
final class $anon extends BinWrapper[shapeless.::[Int,shapeless.::[Boolean,shapeless.HNil]]] {
def <init>() = {
super.<init>();
()
};
def fetch(any: Any): shapeless.::[Int,shapeless.::[Boolean,shapeless.HNil]] = any match {
case (mv @ (_: MapValue)) => mv.getObject match {
case (m @ (_: Map[String, Any])) =>
parse(m).toHList[shapeless.::[Int,shapeless.::[Boolean,shapeless.HNil]]].getOrElse{}
case _ => throw new Exception("Wrong type")
}
case _ => throw new Exception("Wrong type")
}
};
new $anon()
}
})
}imports
def fetch: Any => HL
type HL = Int::Boolean::HNil

55. def writeBin[B](b: B)
(implicit bw: BinWrapper[B]): Bin = bw(b)
val asMap = new BinWrapper[Truck] { }
val asJson = new BinWrapper[Truck] { … }
writeBin("tr1", Truck("truck1", 4,
List(1, 2, 3)))(asMap)
writeBin("tr2", Truck("truck2", 2,
List(7, 8, 9)))(asJson)

56. PS: additional features
+ pitfalls

57. case class Sample(name: String, i: Int)
db.put("intKey", "intBin", 202))
db.put("hListKey", "hListBin",
"abcd" :: 2 :: 3 :: HNil))
db.put("mapKey", "mapBin",
Map(Sample("t1", 3) -> "v1",
Sample("t2", 2) -> "v2",
Sample("t3", 1) -> "v3")))

58. AQL > select * from test.test
[
{
"mapBin": {}
},
{
"intBin": 202
},
{
"hListBin": {
"0": "abcd",
"1": 2,
"2": 3
}
}

59. object CleanUp extends App {
val keys = List("mapKey", "intKey", "hListKey")
val res = Future.traverse(keys)(db.delete)
}

60. object CleanUp extends App {
val keys = List("mapKey", "intKey", "hListKey")
val res = Future.traverse(keys)(db.delete)
}
aql> select * from test.test
[
]

61. aerospike-scala-dsl
https://github.com/DanyMariaLee/aerospike-scala
"ru.tinkoff" % "aerospike-scala" % "1.1.14",
"ru.tinkoff" % "aerospike-scala-proto" % "1.1.14",
"ru.tinkoff" % "aerospike-scala-example" % "1.1.14"

62. CONTACT ME
besselfunction@mail.ru
@besseifunction
DanyMariaLee
@besselfunction