98. Getting Records
ids = redis_sort_results.map {|id| id.to_i}
bonds = Bond.find(ids) note that prices (high
write volume data) come
bond_ids_to_bond = {} from elsewhere (not the
SQL db)
bonds.each do |bond|
bond_ids_to_bond[bond.id] = bond
end
results = ids.map do |id|
bond_ids_to_bond[id]
end
99. Getting From Redis
redis.hset("bonds|2", "values", data.to_json)
raw_json = redis.sort("bond_ids", However, then you have
to worry about keeping
the t wo data stores in
:get => "bonds|*->bid_price", sync. We’ll talk about it
later
:get => "bonds|*->values")
results = raw_json.map do |json|
DataObject.new(JSON.parse(json))
end
105. Use a List
O(1) constant time
complexity to add
O(start + n) for reading
N = 500
size = redis.lpush("bond_trades|1", trade_id)
# roll the index
redis.rpop("bond_trades|1") if size > N
# get results
redis.lrange("bond_trades|1", 0, 49)
107. Using a List
redis.lpush("bond_trades|1|2011-05-19-10",
trade_id)
redis.lrange("bond_trades|1|2011-05-19-10",
0, -1)
results = redis.pipelined do
redis.lrange("bond_trades|1|2011-05-19-10",
0, -1)
redis.lrange("bond_trades|1|2011-05-19-09",
0, -1)
end.flatten
108. Rolling the Index
# when something trades
redis.sadd("bonds_traded|2011-05-19-10",
bond_id)
# cron task to remove old data
traded_ids = redis.smembers(
"bonds_traded|2011-05-19-10")
keys = traded_ids.map do |id|
"bond_trades|#{id}|2011-05-19-10"
end
keys << "bonds_traded|2011-05-19-10"
redis.del(*keys)
109. Using a Sorted Set
# Time based Rolling Index using sorted set
redis.zadd("bond_trades|1", O(log(n)) writes
Time.now.to_i, trade_id) O(log(n) + M) reads
# last 20 trades
redis.zrevrange("bond_trades|1", 0, 20)
# trades in the last hour
redis.zrevrangebyscore("bond_trades|1",
"+inf", 1.hour.ago.to_i)
110. Rolling the Index
# cron task to roll the index
bond_ids = redis.smembers("bond_ids")
remove_since_time = 24.hours.ago.to_i
redis.pipelined do
bond_ids.each do |id|
redis.zremrangebyscore(
"bond_trades|#{id}", "-inf",
remove_since_time)
end
end
111. Or roll on read or
write
redis.zadd("bond_trades|1",
Time.now.to_i, trade_id)
redis.zremrangebyscore("bond_trades|1",
"-inf", 24.hours.ago)
112. Indexing N Values
redis.zadd("highest_follower_counts", 2300, 20)
redis.zadd("lowest_follower_counts", 2300, 20)
# rolling the indexes
# keep the lowest
size = redis.zcard("lowest_follower_counts")
redis.zremrangebyrank("lowest_follower_counts",
N, -1) if size > N
# keep the highest
size = redis.zcard("highest_follower_counts")
redis.zremrangebyrank("highest_follower_counts",
0, size - N) if size > N
126. Restore Each Index
time_int = redis.get("last_bond_trade_indexed").to_i
index_time = Time.at(time_int)
trades = Trade.where(
"created_at > :index_time AND created_at <= :now",
{:index_time => index_time, :now => Time.now})
trades.each do |trade| list you have to run while
not writing new data.
trade.index_in_redis Set can be made to run
end while writing new data
129. Sets don’t work with
intersection, union, or
diff.
SORT won’t work unless
all those keys fall on the
same server
Easy to Scale
(consistent hashing)
133. Index the minimum to
keep memory footprint
down
use rolling indexes, don’t
keep more shit in
memory than you need.
Users won’t page through
20 pages of results, so
don’t store that many