From efc052cd99d28bb422aa8e02bdfe498d0eb5f7a4 Mon Sep 17 00:00:00 2001 From: Ivan Remen Date: Sat, 21 Jan 2017 01:22:53 +0300 Subject: [PATCH 1/9] Helpers api in profile --- profiles.lua | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/profiles.lua b/profiles.lua index 378b343..488683b 100644 --- a/profiles.lua +++ b/profiles.lua @@ -502,3 +502,69 @@ function profile_print_bigger_than(size_in_bytes) end end end + +local helper_key_id = 1011 +local email_key_id = 314 + +local function get_helpers(user_id, helpers_string) + if (helpers_string:len() - 1) % 4 ~= 0 then + return nil, "Bad size of helpers" + end + + local helpers = {} + local i = 1 + while i < helpers_string:len() do + local index = box.unpack('i', string.sub(helpers_string, i, i + 3)) + local state = bit.band(index, 0x80000000) + index = bit.band(index, 0x7FFFFFFF) + i = i + 4 + + local time = box.unpack('i', string.sub(helpers_string, i, i + 3)) + i = i + 4 + + local show = box.unpack('i', string.sub(helpers_string, i, i + 3)) + i = i + 4 + + local close = box.unpack('i', string.sub(helpers_string, i, i + 3)) + i = i + 4 + + local helper = {index = index, state = state, time = time, show = show, close = close} + + helpers[index] = helper + end + + if string.sub(helpers_string, i, i+1) ~= '!' then + return nil, "Non '!' end of helpers, invalid helpers" + end + + return helpers, nil +end + +function profile_print_with_helper(helper_number) + if box.cfg.replication_source == nil then error("replica api only") end + if type(helper_number) == "string" then helper_number = tonumber(helper_number) end + + profile_apply_func( + function(p, helper_number) + local packed_key_id = box.pack("w", helper_key_id) + local packed_email_key_id = box.pack("w", email_key_id) + + if p.prefs[packed_key_id] then + if p.prefs[packed_key_id]:len() > 1 then + local helpers, err = get_helpers(profile_id_to_int(p.id), p.prefs[packed_key_id]) + + if helpers then + if helpers[helper_number] then + print("user_id: ", profile_id_to_int(p.id), " id: ", helper_key_id, " val: ", helper_number, " ", helpers[helper_number]['state'], " ", helpers[helper_number]['time'], " ", helpers[helper_number]['show'], " ", helpers[helper_number]['close'], " ", p.prefs[packed_email_key_id]) + end + else + print("user_id: ", profile_id_to_int(p.id), " id: ", helper_key_id, " val: BAD (", err, ")") + end + end + end + end, + helper_number + ) + +end + From d784fd75071215fe24ac946fea716d6e2d68c69b Mon Sep 17 00:00:00 2001 From: Ivan Remen Date: Sat, 21 Jan 2017 02:30:10 +0300 Subject: [PATCH 2/9] Delete fake parametr --- profiles.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/profiles.lua b/profiles.lua index 488683b..2f16fdd 100644 --- a/profiles.lua +++ b/profiles.lua @@ -506,7 +506,7 @@ end local helper_key_id = 1011 local email_key_id = 314 -local function get_helpers(user_id, helpers_string) +local function get_helpers(helpers_string) if (helpers_string:len() - 1) % 4 ~= 0 then return nil, "Bad size of helpers" end @@ -551,7 +551,7 @@ function profile_print_with_helper(helper_number) if p.prefs[packed_key_id] then if p.prefs[packed_key_id]:len() > 1 then - local helpers, err = get_helpers(profile_id_to_int(p.id), p.prefs[packed_key_id]) + local helpers, err = get_helpers(p.prefs[packed_key_id]) if helpers then if helpers[helper_number] then From 1be9181f8e6e97a0726687afcf4eceb0573f58a5 Mon Sep 17 00:00:00 2001 From: Ivan Remen Date: Mon, 10 Apr 2017 07:08:47 -0400 Subject: [PATCH 3/9] Ussender clear --- ussender.lua | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ussender.lua b/ussender.lua index d3c8502..e4ac3da 100644 --- a/ussender.lua +++ b/ussender.lua @@ -1,12 +1,28 @@ -- Race conditions in this files are possible. It is ok by biz logic. +-- + +local tuple_length_limit = 1001 --max numbers of element + 1. VALUE MUST BE ODD. + +local function truncate_tuple(selected) + selected = selected:transform(1, #selected - tuple_length_limit):transform(1, (tuple_length_limit - 1) / 2) + box.replace(0, selected) + return selected +end + function ussender_add(user_id, sender_id) local user_id = box.unpack("i", user_id) local sender_id = box.unpack("l", sender_id) local selected = { box.select_limit(0, 0, 0, 1, user_id) } + if #selected == 0 then box.insert(0, user_id, sender_id) else + + if #selected[1] >= tuple_length_limit then + selected[1] = truncate_tuple(selected[1]) + end + local fun, param, state = selected[1]:pairs() state, _ = fun(param, state) -- skip the first element of tuple for state, v in fun, param, state do @@ -26,6 +42,9 @@ function ussender_select(user_id) if #ret == 0 then return {user_id} end + if #ret[1] >= tuple_length_limit then + ret[1] = truncate_tuple(ret[1]) + end return ret end From f4e5f09c9ecb772bdd2120c7d03a9023d063316d Mon Sep 17 00:00:00 2001 From: Ivan Remen Date: Mon, 10 Apr 2017 07:39:11 -0400 Subject: [PATCH 4/9] space number unhardcode --- ussender.lua | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ussender.lua b/ussender.lua index e4ac3da..dbf914d 100644 --- a/ussender.lua +++ b/ussender.lua @@ -2,10 +2,11 @@ -- local tuple_length_limit = 1001 --max numbers of element + 1. VALUE MUST BE ODD. +local space_no = 0 local function truncate_tuple(selected) selected = selected:transform(1, #selected - tuple_length_limit):transform(1, (tuple_length_limit - 1) / 2) - box.replace(0, selected) + box.replace(space_no, selected) return selected end @@ -13,7 +14,7 @@ function ussender_add(user_id, sender_id) local user_id = box.unpack("i", user_id) local sender_id = box.unpack("l", sender_id) - local selected = { box.select_limit(0, 0, 0, 1, user_id) } + local selected = { box.select_limit(space_no, 0, 0, 1, user_id) } if #selected == 0 then box.insert(0, user_id, sender_id) @@ -31,14 +32,14 @@ function ussender_add(user_id, sender_id) return end end - box.update(0, user_id, "!p", -1, sender_id) + box.update(space_no, user_id, "!p", -1, sender_id) end end function ussender_select(user_id) local user_id = box.unpack("i", user_id) - local ret = {box.select_limit(0, 0, 0, 1, user_id)} + local ret = {box.select_limit(space_no, 0, 0, 1, user_id)} if #ret == 0 then return {user_id} end @@ -50,5 +51,5 @@ end function ussender_delete(user_id) local user_id = box.unpack("i", user_id) - box.delete(0, user_id) + box.delete(space_no, user_id) end From 6111814e6ac2d0415cf5b29f1735737259855028 Mon Sep 17 00:00:00 2001 From: Ivan Remen Date: Mon, 10 Apr 2017 07:40:17 -0400 Subject: [PATCH 5/9] missed space_no --- ussender.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ussender.lua b/ussender.lua index dbf914d..e4a901f 100644 --- a/ussender.lua +++ b/ussender.lua @@ -17,7 +17,7 @@ function ussender_add(user_id, sender_id) local selected = { box.select_limit(space_no, 0, 0, 1, user_id) } if #selected == 0 then - box.insert(0, user_id, sender_id) + box.insert(space_no, user_id, sender_id) else if #selected[1] >= tuple_length_limit then From c9e6acfeb796d5cdae8ecfd5b9789a6952dc3606 Mon Sep 17 00:00:00 2001 From: Ivan Remen Date: Thu, 27 Jul 2017 08:09:34 -0400 Subject: [PATCH 6/9] Profile 1.7 initial --- profile17.lua | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 profile17.lua diff --git a/profile17.lua b/profile17.lua new file mode 100644 index 0000000..43f15c0 --- /dev/null +++ b/profile17.lua @@ -0,0 +1,84 @@ +box.cfg{listen = 3301} +box.once('init', function() + profile_space = box.schema.space.create('profile', {id = 1}) + profile_space:create_index('primary', { + type = 'hash', + parts = {1, 'unsigned'} + }) + box.schema.user.grant('guest', 'read,write,execute', 'universe') + end) + +log = require('log') + +local function store_profile(profile) + return box.space.profile:replace({profile.uid, profile.data}) +end + +local function load_profile(user_id) + local tup = box.space.profile:select(user_id) + if #tup == 0 then + return nil + end + local profile = {} + profile.uid = user_id + profile.data = tup[1][2] + return profile +end + +local function create_new_profile(user_id) + local profile = {} + profile.uid = user_id + profile.data = {} + return profile +end + +local function set_profile_key(profile, key, value) + if value == '' then + value = nil + end + profile.data[key] = value +end + +local function cast_profile_to_return_format(profile) + return {profile.uid, profile.data} +end + +function profile_delete(user_id) + box.space.profile:delete(user_id) +end + +function profile_get_all(user_id) + local profile = load_profile(user_id) + return cast_profile_to_return_format(profile) +end + +function profile_multiset(user_id, ...) + local pref_list = {...} + if #pref_list == 0 then + error('No keys passed') + end + + if #pref_list % 2 ~= 0 then + error('Not even number of arguments') + end + + local profile = load_profile(user_id) + if not profile then + profile = create_new_profile(user_id) + end + + local i, pref_key = next(pref_list) + local i, pref_value = next(pref_list, i) + while pref_key ~= nil and pref_value ~= nil do + set_profile_key(profile, pref_key, pref_value) + i, pref_key = next(pref_list, i) + i, pref_value = next(pref_list, i) + end + + store_profile(profile) + return cast_profile_to_return_format(profile) +end + +function profile_set(user_id, key, value) + return profile_multiset(user_id, key, value) +end From 3cd3b2381ac629ef8eb1c65a888b7ba099c8579b Mon Sep 17 00:00:00 2001 From: Ivan Remen Date: Thu, 27 Jul 2017 19:08:21 -0400 Subject: [PATCH 7/9] Fixes and new features --- profile17.lua | 82 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/profile17.lua b/profile17.lua index 43f15c0..00de9d4 100644 --- a/profile17.lua +++ b/profile17.lua @@ -1,74 +1,113 @@ +-- Tuple of profile has always 2 fields. +-- Field 1 -- user-id +-- Field 2 -- key/value dict. Key is always a number. + box.cfg{listen = 3301} box.once('init', function() profile_space = box.schema.space.create('profile', {id = 1}) profile_space:create_index('primary', { - type = 'hash', - parts = {1, 'unsigned'} + type = 'tree', + parts = {1, 'unsigned'}, + unique = true }) box.schema.user.grant('guest', 'read,write,execute', 'universe') end) -log = require('log') +msgpack = require('msgpack') + +-- Cast internal profile format to the format, requested by client-side. +local function cast_profile_to_return_format(profile) + setmetatable(profile.data, msgpack.map_mt) -- If we doesnt call this, we will receive an array with many nulls, not map. + return {profile.uid, profile.data} +end local function store_profile(profile) + -- We dont want to store empty profiles. Save space. + if #profile.data == 0 then + return box.space.profile:delete(profile.uid) + end + + setmetatable(profile.data, msgpack.map_mt) -- If we doesnt call this, we will store an array with many nulls, not map. return box.space.profile:replace({profile.uid, profile.data}) end -local function load_profile(user_id) - local tup = box.space.profile:select(user_id) - if #tup == 0 then - return nil - end +local function create_new_profile(user_id) local profile = {} profile.uid = user_id - profile.data = tup[1][2] + profile.data = {} return profile end -local function create_new_profile(user_id) +local function load_profile(user_id) + local tup = box.space.profile:select(user_id) + -- In case no profile found, operate it as profile without keys/values + if #tup == 0 then + return create_new_profile(user_id) + end local profile = {} profile.uid = user_id - profile.data = {} + profile.data = tup[1][2] -- Index 1 is because we have only 1 tuple with such userid (index is unique). Second field of tuple is key/value dict. return profile end local function set_profile_key(profile, key, value) + -- Do not store empty keys. We want to save space. if value == '' then value = nil end profile.data[key] = value end -local function cast_profile_to_return_format(profile) - return {profile.uid, profile.data} -end - +-- function profile_delete delete profile. Returns nothing function profile_delete(user_id) box.space.profile:delete(user_id) end +-- function profile_get_all returns full profile function profile_get_all(user_id) local profile = load_profile(user_id) return cast_profile_to_return_format(profile) end -function profile_multiset(user_id, ...) +-- function profile_multiget returns only requested keys from profile. Accepts user_id and then several keys +function profile_multiget(user_id, ...) + -- First of all, make hash of needed keys local pref_list = {...} - if #pref_list == 0 then - error('No keys passed') + local pref_hash = {} + for k, v in ipairs(pref_list) do + pref_hash[v] = true end + local profile = load_profile(user_id) + -- Create a copy of profile. We select few keys, so it is faster to copy only needed keys, then clear not needed keys + local profile_copy = create_new_profile(profile.uid) + for k, v in pairs(profile.data) do + if pref_hash[k] then + profile_copy.data[k] = v + end + end + + return cast_profile_to_return_format(profile_copy) +end + +-- function profile_multiset accepts user_id and then key, value, key, value, key, value, ... Returns full updated profile. +function profile_multiset(user_id, ...) + local pref_list = {...} + if #pref_list % 2 ~= 0 then error('Not even number of arguments') end - + local profile = load_profile(user_id) - if not profile then - profile = create_new_profile(user_id) + + -- In case of no keys were passed, just return full profile + if #pref_list == 0 then + return cast_profile_to_return_format(profile) end local i, pref_key = next(pref_list) local i, pref_value = next(pref_list, i) + -- iterate all passed key/value pairs from arguments while pref_key ~= nil and pref_value ~= nil do set_profile_key(profile, pref_key, pref_value) i, pref_key = next(pref_list, i) @@ -79,6 +118,7 @@ function profile_multiset(user_id, ...) return cast_profile_to_return_format(profile) end +-- function profile_set set only one key. Returns full updated profile function profile_set(user_id, key, value) return profile_multiset(user_id, key, value) end From d539abdf5933064d65ff0292d7bdcd92c1809e92 Mon Sep 17 00:00:00 2001 From: Ivan Remen Date: Fri, 28 Jul 2017 07:30:32 -0400 Subject: [PATCH 8/9] Authable access --- profile17.lua | 85 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 19 deletions(-) diff --git a/profile17.lua b/profile17.lua index 00de9d4..776f016 100644 --- a/profile17.lua +++ b/profile17.lua @@ -3,31 +3,76 @@ -- Field 2 -- key/value dict. Key is always a number. box.cfg{listen = 3301} -box.once('init', function() - profile_space = box.schema.space.create('profile', {id = 1}) - profile_space:create_index('primary', { - type = 'tree', - parts = {1, 'unsigned'}, - unique = true - }) - box.schema.user.grant('guest', 'read,write,execute', 'universe') - end) + +-- Create a user, then init storage, create functions and then revoke ALL priveleges from user +local function init_storage(init_func, interface) + local init_username = "profile" + box.schema.user.create(init_username, {if_not_exists = true}) + box.schema.user.grant(init_username, 'execute,read,write', 'universe', nil, + {if_not_exists = true}) + box.session.su(init_username) + + init_func() + + for _, v in pairs(interface) do + box.schema.func.create(v, {setuid = true, if_not_exists = true}) + end + + box.session.su('admin') + box.schema.user.revoke(init_username, 'execute,read,write', 'universe') +end + +-- Create a role, which can execute interface functions +local function init_role(role_name, interface) + box.schema.role.create(role_name, {if_not_exists = true}) + for _, v in pairs(interface) do + box.schema.role.grant(role_name, 'execute', 'function', v, + {if_not_exists = true}) + end +end + +-- Scheme initialization +local function init() + local s = box.schema.create_space('profile', { + id = 0, + if_not_exists = true, + }) + s:create_index('primary', { + id = 0, + type = 'tree', + unique = true, + parts = {1, 'unsigned'}, + if_not_exists = true, + }) +end + +-- List of functions, which is possible to call from outside the box +local interface = { + 'profile_delete', + 'profile_get_all', + 'profile_multiget', + 'profile_multiset', + 'profile_set', +} msgpack = require('msgpack') -- Cast internal profile format to the format, requested by client-side. local function cast_profile_to_return_format(profile) - setmetatable(profile.data, msgpack.map_mt) -- If we doesnt call this, we will receive an array with many nulls, not map. + setmetatable(profile.data, msgpack.map_mt) return {profile.uid, profile.data} end local function store_profile(profile) + local count = 0 + for k,v in pairs(profile.data) do count = count + 1 end + -- We dont want to store empty profiles. Save space. - if #profile.data == 0 then + if count == 0 then return box.space.profile:delete(profile.uid) end - setmetatable(profile.data, msgpack.map_mt) -- If we doesnt call this, we will store an array with many nulls, not map. + setmetatable(profile.data, msgpack.map_mt) return box.space.profile:replace({profile.uid, profile.data}) end @@ -71,19 +116,18 @@ end -- function profile_multiget returns only requested keys from profile. Accepts user_id and then several keys function profile_multiget(user_id, ...) - -- First of all, make hash of needed keys local pref_list = {...} - local pref_hash = {} - for k, v in ipairs(pref_list) do - pref_hash[v] = true + if #pref_list == 0 then + return cast_profile_to_return_format(create_new_profile(user_id)) end local profile = load_profile(user_id) + -- Create a copy of profile. We select few keys, so it is faster to copy only needed keys, then clear not needed keys local profile_copy = create_new_profile(profile.uid) - for k, v in pairs(profile.data) do - if pref_hash[k] then - profile_copy.data[k] = v + for _, v in ipairs(pref_list) do + if profile.data[v] then + profile_copy.data[v] = profile.data[v] end end @@ -122,3 +166,6 @@ end function profile_set(user_id, key, value) return profile_multiset(user_id, key, value) end + +init_storage(init, interface) +init_role('profile_role', interface) From bf137fd8e998629ec68327e6d66295c2dc4bf038 Mon Sep 17 00:00:00 2001 From: Ivan Remen Date: Sun, 30 Jul 2017 13:34:12 -0400 Subject: [PATCH 9/9] Proscribe client added --- profile17.lua | 44 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/profile17.lua b/profile17.lua index 776f016..313cb81 100644 --- a/profile17.lua +++ b/profile17.lua @@ -2,11 +2,13 @@ -- Field 1 -- user-id -- Field 2 -- key/value dict. Key is always a number. -box.cfg{listen = 3301} +proscribe_host = '10.161.41.111' +proscribe_port = 27017 +keys_to_proscribe = {324} -- Create a user, then init storage, create functions and then revoke ALL priveleges from user local function init_storage(init_func, interface) - local init_username = "profile" + local init_username = 'profile' box.schema.user.create(init_username, {if_not_exists = true}) box.schema.user.grant(init_username, 'execute,read,write', 'universe', nil, {if_not_exists = true}) @@ -34,11 +36,9 @@ end -- Scheme initialization local function init() local s = box.schema.create_space('profile', { - id = 0, if_not_exists = true, }) s:create_index('primary', { - id = 0, type = 'tree', unique = true, parts = {1, 'unsigned'}, @@ -56,6 +56,41 @@ local interface = { } msgpack = require('msgpack') +socket = require('socket') +digest = require('digest') +pickle = require('pickle') + +proscribe_socket = socket('AF_INET', 'SOCK_DGRAM', 'udp') + +-- Function, which sends requested profile_key changes to special DWH daemon +local function send_to_proscribe(uid, profile_key, old_value, new_value) + local fit = false + for _, v in ipairs(keys_to_proscribe) do + if v == profile_key then + fit = true + end + end + if not fit then + return + end + + if not old_value then + old_value = '' + end + if not new_value then + new_value = '' + end + if old_value == new_value then -- we doesnt want to send data without changes + return + end + + old_value = digest.base64_encode(old_value) + new_value = digest.base64_encode(new_value) + + local body = pickle.pack('iliiAiA', 1, uid, profile_key, #old_value, old_value, #new_value, new_value) + local packet = pickle.pack('iiiA', 1, #body, 0, body) + proscribe_socket:sendto(proscribe_host, proscribe_port, packet) +end -- Cast internal profile format to the format, requested by client-side. local function cast_profile_to_return_format(profile) @@ -153,6 +188,7 @@ function profile_multiset(user_id, ...) local i, pref_value = next(pref_list, i) -- iterate all passed key/value pairs from arguments while pref_key ~= nil and pref_value ~= nil do + send_to_proscribe(profile.uid, pref_key, profile.data[pref_key], pref_value) set_profile_key(profile, pref_key, pref_value) i, pref_key = next(pref_list, i) i, pref_value = next(pref_list, i)