Model Upload Notification for Redis Replicas #390
Description
When running the command OCTO.UPLOAD {file_name} on a Redis replica, the following error occurs:
== CRITICAL == This replica is sending an error to its master: '-only master can upload model' after processing the command 'OCTO.UPLOAD'
Latest backlog is: '"\n$18\r\n__sentinel__:hello\r\n$87\r\n10.1.14.4,36379,94668a13ba037b093bba73fae61e333cff0ed95f,89,master01,10.1.8.235,6379,89\r\n*3\r\n$7\r\nPUBLISH\r\n$18\r\n__sentinel__:hello\r\n$87\r\n10.1.14.4,36379,94668a13ba037b093bba73fae61e333cff0ed95f,89,master01,10.1.8.235,6379,89\r\n"'
Code:
fn upload_model(ctx: &Context, args: Vec<RedisString>) -> RedisResult {
if args.len() != 2 {
return Err(RedisError::WrongArity);
};
if !ctx.get_flags().contains(ContextFlags::MASTER) {
return Err(RedisError::Str("only master can upload model"));
}
let mut args = args.into_iter().skip(1);
let file_name = args.next_string()?;
ctx.replicate_verbatim();
let key = RedisString::create(NonNull::new(ctx.ctx), file_name.as_str());
let status = ctx.notify_keyspace_event(NotifyEvent::GENERIC, "model.upload", &key);
match status {
Status::Ok => REDIS_OK,
Status::Err => Err(RedisError::Str("fail send event")),
}
}
fn event_upload_model(ctx: &Context, event_type: NotifyEvent, event: &str, key: &[u8]) {
let msg = format!(
"Received event: {:?} on key: {} via event: {}",
event_type,
std::str::from_utf8(key).unwrap(),
event
);
let file_name = std::str::from_utf8(key).unwrap().to_string();
ctx.log_notice(msg.as_str());
std::thread::spawn(move || {
match catboost::upload_catboost_model(
std::env::temp_dir().join(format!("models/{}", file_name)),
) {
Ok(model) => {
*catboost::CATBOOST_MODEL.lock().unwrap() = model;
*catboost::CATBOOST_VERSION.lock().unwrap() = file_name.clone();
}
Err(err) => {
error!(
"Error updating CatBoost model with file '{}': {}",
file_name.to_string().clone(),
err
);
}
};
});
}
redis_module! {
name: crate::MODULE_NAME,
version: 1,
allocator: (get_allocator!(), get_allocator!()),
data_types: [CREATIVE_CONDITION],
init:init,
commands: [
["OCTO.UPLOAD", upload_model, "readonly", 1, 1, 1],
],
event_handlers: [
[@GENERIC: event_upload_model],
],
}
How can I perform a model update on replicas using notifications?
Redis version=7.2.4
redis-module = "2.0.7
Docker-compose
services:
redis-master:
image: redis:latest
container_name: redis-master
hostname: redis-master
ports:
- "6379:6379"
volumes:
- ./output/libsentinel.so:/usr/local/libsentinel.so
- ./data/master:/data
command:
[
"redis-server",
"--appendonly","yes",
"--loadmodule", "/usr/local/libsentinel.so",
"--repl-diskless-load","on-empty-db",
"--replica-announce-ip","${HOST_IP}",
"--replica-announce-port","6379",
"--protected-mode","no"
]
networks:
redis-net:
ipv4_address: 172.21.0.3
redis-slave-1:
image: redis:latest
container_name: redis-slave-1
hostname: redis-slave-1
depends_on:
- redis-master
ports:
- "6380:6379"
volumes:
- ./output/libsentinel.so:/usr/local/libsentinel.so
- ./data/slave1:/data
command:
[
"redis-server",
"--appendonly","yes",
"--loadmodule", "/usr/local/libsentinel.so",
"--replicaof","redis-master","6379",
"--repl-diskless-load", "on-empty-db",
"--replica-announce-ip", "${HOST_IP}",
"--replica-announce-port", "6380",
"--protected-mode", "no"
]
networks:
redis-net:
ipv4_address: 172.21.0.4
redis-slave-2:
image: redis:latest
container_name: redis-slave-2
hostname: redis-slave-2
depends_on:
- redis-master
ports:
- "6381:6379"
volumes:
- ./output/libsentinel.so:/usr/local/libsentinel.so
- ./data/slave2:/data
command:
[
"redis-server",
"--appendonly",
"yes",
"--loadmodule", "/usr/local/libsentinel.so",
"--replicaof",
"redis-master",
"6379",
"--repl-diskless-load",
"on-empty-db",
"--replica-announce-ip",
"${HOST_IP}",
"--replica-announce-port",
"6381",
"--protected-mode",
"no"
]
networks:
redis-net:
ipv4_address: 172.21.0.5
sentinel-1:
image: redis:latest
container_name: sentinel-1
hostname: sentinel-1
depends_on:
- redis-master
ports:
- "26379:26379"
command: >
sh -c 'echo "bind 0.0.0.0" > /etc/sentinel.conf &&
echo "sentinel monitor mymaster ${HOST_IP} 6379 2" >> /etc/sentinel.conf &&
echo "sentinel resolve-hostnames yes" >> /etc/sentinel.conf &&
echo "sentinel down-after-milliseconds mymaster 10000" >> /etc/sentinel.conf &&
echo "sentinel failover-timeout mymaster 10000" >> /etc/sentinel.conf &&
echo "sentinel parallel-syncs mymaster 1" >> /etc/sentinel.conf &&
redis-sentinel /etc/sentinel.conf'
networks:
redis-net:
ipv4_address: 172.21.0.6
sentinel-2:
image: redis:latest
container_name: sentinel-2
hostname: sentinel-2
depends_on:
- redis-master
ports:
- "26380:26379"
command: >
sh -c 'echo "bind 0.0.0.0" > /etc/sentinel.conf &&
echo "sentinel monitor mymaster ${HOST_IP} 6379 2" >> /etc/sentinel.conf &&
echo "sentinel resolve-hostnames yes" >> /etc/sentinel.conf &&
echo "sentinel down-after-milliseconds mymaster 10000" >> /etc/sentinel.conf &&
echo "sentinel failover-timeout mymaster 10000" >> /etc/sentinel.conf &&
echo "sentinel parallel-syncs mymaster 1" >> /etc/sentinel.conf &&
redis-sentinel /etc/sentinel.conf'
networks:
redis-net:
ipv4_address: 172.21.0.7
sentinel-3:
image: redis:latest
container_name: sentinel-3
hostname: sentinel-3
depends_on:
- redis-master
ports:
- "26381:26379"
command: >
sh -c 'echo "bind 0.0.0.0" > /etc/sentinel.conf &&
echo "sentinel monitor mymaster ${HOST_IP} 6379 2" >> /etc/sentinel.conf &&
echo "sentinel resolve-hostnames yes" >> /etc/sentinel.conf &&
echo "sentinel down-after-milliseconds mymaster 10000" >> /etc/sentinel.conf &&
echo "sentinel failover-timeout mymaster 10000" >> /etc/sentinel.conf &&
echo "sentinel parallel-syncs mymaster 1" >> /etc/sentinel.conf &&
redis-sentinel /etc/sentinel.conf'
networks:
redis-net:
ipv4_address: 172.21.0.8
networks:
redis-net:
driver: bridge
ipam:
config:
- subnet: 172.21.0.0/16