From e83aa79380d7b167281a23b79fc709139c53468c Mon Sep 17 00:00:00 2001 From: koushiro Date: Tue, 23 Dec 2025 15:09:22 +0800 Subject: [PATCH 1/2] refactor: Split redis service to new crate --- core/Cargo.lock | 29 +++++++-------- core/Cargo.toml | 28 +++++---------- core/core/Cargo.toml | 22 +----------- core/core/src/services/mod.rs | 5 --- core/services/redis/Cargo.toml | 36 +++++++++++++++++++ .../redis => services/redis/src}/backend.rs | 4 +-- .../redis => services/redis/src}/config.rs | 29 ++++++++------- .../redis => services/redis/src}/core.rs | 4 +-- .../redis => services/redis/src}/delete.rs | 12 ++++--- .../redis => services/redis/src}/docs.md | 4 +-- .../mod.rs => services/redis/src/lib.rs} | 11 +++--- .../redis => services/redis/src}/writer.rs | 11 +++--- core/src/lib.rs | 4 +++ 13 files changed, 106 insertions(+), 93 deletions(-) create mode 100644 core/services/redis/Cargo.toml rename core/{core/src/services/redis => services/redis/src}/backend.rs (99%) rename core/{core/src/services/redis => services/redis/src}/config.rs (91%) rename core/{core/src/services/redis => services/redis/src}/core.rs (99%) rename core/{core/src/services/redis => services/redis/src}/delete.rs (87%) rename core/{core/src/services/redis => services/redis/src}/docs.md (94%) rename core/{core/src/services/redis/mod.rs => services/redis/src/lib.rs} (84%) rename core/{core/src/services/redis => services/redis/src}/writer.rs (91%) diff --git a/core/Cargo.lock b/core/Cargo.lock index 9f9e8222620b..852e81b01d52 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -5539,11 +5539,7 @@ version = "0.55.0" dependencies = [ "anyhow", "bytes", - "criterion", "divan", - "dotenvy", - "fastrace", - "fastrace-jaeger", "futures", "http 1.4.0", "libtest-mimic", @@ -5617,6 +5613,7 @@ dependencies = [ "opendal-service-persy", "opendal-service-postgresql", "opendal-service-redb", + "opendal-service-redis", "opendal-service-rocksdb", "opendal-service-s3", "opendal-service-seafile", @@ -5633,16 +5630,11 @@ dependencies = [ "opendal-service-webhdfs", "opendal-service-yandex-disk", "opendal-testkit", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry_sdk", - "pretty_assertions", "rand 0.8.5", "reqwest", "sha2", "size", "tokio", - "tracing-opentelemetry", "tracing-subscriber", "uuid", ] @@ -5678,13 +5670,10 @@ name = "opendal-core" version = "0.55.0" dependencies = [ "anyhow", - "backon", "base64 0.22.1", "bytes", "ctor", - "fastpool", "futures", - "getrandom 0.2.16", "http 1.4.0", "http-body 1.0.1", "jiff", @@ -5696,8 +5685,6 @@ dependencies = [ "pretty_assertions", "quick-xml", "rand 0.8.5", - "redis", - "reqsign", "reqwest", "serde", "serde_json", @@ -6617,6 +6604,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "opendal-service-redis" +version = "0.55.0" +dependencies = [ + "bytes", + "ctor", + "fastpool", + "http 1.4.0", + "opendal-core", + "redis", + "serde", + "tokio", +] + [[package]] name = "opendal-service-rocksdb" version = "0.55.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index 3a301d73d39b..5a6d1cc3fb75 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -160,8 +160,8 @@ services-pcloud = ["dep:opendal-service-pcloud"] services-persy = ["dep:opendal-service-persy"] services-postgresql = ["dep:opendal-service-postgresql"] services-redb = ["dep:opendal-service-redb"] -services-redis = ["opendal-core/services-redis"] -services-redis-native-tls = ["opendal-core/services-redis-native-tls"] +services-redis = ["opendal-service-redis?/rustls"] +services-redis-native-tls = ["opendal-service-redis?/native-tls"] services-rocksdb = ["dep:opendal-service-rocksdb"] services-s3 = ["dep:opendal-service-s3"] services-seafile = ["dep:opendal-service-seafile"] @@ -266,6 +266,7 @@ opendal-service-pcloud = { path = "services/pcloud", version = "0.55.0", optiona opendal-service-persy = { path = "services/persy", version = "0.55.0", optional = true, default-features = false } opendal-service-postgresql = { path = "services/postgresql", version = "0.55.0", optional = true, default-features = false } opendal-service-redb = { path = "services/redb", version = "0.55.0", optional = true, default-features = false } +opendal-service-redis = { path = "services/redis", version = "0.55.0", optional = true, default-features = false } opendal-service-rocksdb = { path = "services/rocksdb", version = "0.55.0", optional = true, default-features = false } opendal-service-s3 = { path = "services/s3", version = "0.55.0", optional = true, default-features = false } opendal-service-seafile = { path = "services/seafile", version = "0.55.0", optional = true, default-features = false } @@ -283,35 +284,24 @@ opendal-service-webhdfs = { path = "services/webhdfs", version = "0.55.0", optio opendal-service-yandex-disk = { path = "services/yandex-disk", version = "0.55.0", optional = true, default-features = false } opendal-testkit = { path = "testkit", version = "0.55.0", optional = true } +[target.'cfg(target_arch = "wasm32")'.dependencies] +opendal-service-opfs = { path = "services/opfs", version = "0.55.0", optional = true, default-features = false } + [dev-dependencies] -anyhow = { version = "1.0.100", features = ["std"] } +anyhow = "1.0" bytes = { workspace = true } -criterion = { version = "0.7", features = ["async", "async_tokio"] } -divan = { version = "0.1" } -dotenvy = "0.15" -fastrace = { version = "0.7", features = ["enable"] } -fastrace-jaeger = "0.7" +divan = "0.1" futures = { workspace = true, default-features = true } http = { workspace = true } libtest-mimic = "0.8" log = { workspace = true } -opentelemetry = { version = "0.31.0", default-features = false, features = [ - "trace", -] } -opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic"] } -opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio"] } -pretty_assertions = "1" rand = { workspace = true } -reqwest = { version = "0.12.24" } +reqwest = "0.12.24" sha2 = { workspace = true } size = "0.5" tokio = { workspace = true, features = ["fs", "macros", "rt-multi-thread"] } -tracing-opentelemetry = "0.32.0" tracing-subscriber = { version = "0.3", features = [ "env-filter", "tracing-log", ] } uuid = { workspace = true, features = ["serde", "v4"] } - -[target.'cfg(target_arch = "wasm32")'.dependencies] -opendal-service-opfs = { path = "services/opfs", version = "0.55.0", optional = true, default-features = false } diff --git a/core/core/Cargo.toml b/core/core/Cargo.toml index e372b3b2f607..650628588cee 100644 --- a/core/core/Cargo.toml +++ b/core/core/Cargo.toml @@ -55,8 +55,6 @@ internal-tokio-rt = ["tokio/rt-multi-thread"] executors-tokio = ["tokio/rt"] services-memory = [] -services-redis = ["dep:redis", "dep:fastpool", "redis?/tokio-rustls-comp"] -services-redis-native-tls = ["services-redis", "redis?/tokio-native-tls-comp"] [lib] bench = false @@ -87,27 +85,9 @@ url = "2.5" uuid = { workspace = true, features = ["serde", "v4"] } # Optional dependencies - -# Services -# general dependencies. -fastpool = { version = "1.0.2", optional = true } -sha2 = { workspace = true, optional = true } - -# For http based services. -reqsign = { workspace = true, features = ["reqwest_request"], optional = true } - -# for services-moka -moka = { version = "0.12", optional = true, features = ["future", "sync"] } -# for services-redis -redis = { version = "1.0", features = [ - "cluster-async", - "tokio-comp", - "connection-manager", -], optional = true } +moka = { version = "0.12", optional = true, features = ["sync"] } [target.'cfg(target_arch = "wasm32")'.dependencies] -backon = { version = "1.6", features = ["gloo-timers-sleep"] } -getrandom = { version = "0.2", features = ["js"] } jiff = { version = "0.2.15", features = ["serde", "js"] } tokio = { workspace = true, features = ["time"] } uuid = { workspace = true, features = ["serde", "v4", "js"] } diff --git a/core/core/src/services/mod.rs b/core/core/src/services/mod.rs index ec77723ab1ee..f7c8457e0541 100644 --- a/core/core/src/services/mod.rs +++ b/core/core/src/services/mod.rs @@ -23,8 +23,3 @@ mod memory; #[cfg(feature = "services-memory")] pub use self::memory::*; - -#[cfg(feature = "services-redis")] -mod redis; -#[cfg(feature = "services-redis")] -pub use self::redis::*; diff --git a/core/services/redis/Cargo.toml b/core/services/redis/Cargo.toml new file mode 100644 index 000000000000..f8587a98ebe2 --- /dev/null +++ b/core/services/redis/Cargo.toml @@ -0,0 +1,36 @@ +[package] +description = "Apache OpenDAL Redis service implementation" +name = "opendal-service-redis" + +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[package.metadata.docs.rs] +all-features = true + +[features] +default = ["rustls"] +native-tls = ["redis/tokio-native-tls-comp"] +rustls = ["redis/tokio-rustls-comp"] + +[dependencies] +bytes = { workspace = true } +ctor = { workspace = true } +fastpool = "1.0.2" +http = { workspace = true } +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } +redis = { version = "1.0", features = [ + "cluster-async", + "tokio-comp", + "connection-manager", +] } +serde = { workspace = true, features = ["derive"] } +tokio = { workspace = true, features = ["macros", "time"] } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/core/core/src/services/redis/backend.rs b/core/services/redis/src/backend.rs similarity index 99% rename from core/core/src/services/redis/backend.rs rename to core/services/redis/src/backend.rs index a23f303bb2ed..6d809a504fea 100644 --- a/core/core/src/services/redis/backend.rs +++ b/core/services/redis/src/backend.rs @@ -19,6 +19,8 @@ use std::path::PathBuf; use std::sync::Arc; use http::Uri; +use opendal_core::raw::*; +use opendal_core::*; use redis::Client; use redis::ConnectionAddr; use redis::ConnectionInfo; @@ -32,8 +34,6 @@ use super::config::RedisConfig; use super::core::*; use super::delete::RedisDeleter; use super::writer::RedisWriter; -use crate::raw::*; -use crate::*; const DEFAULT_REDIS_ENDPOINT: &str = "tcp://127.0.0.1:6379"; const DEFAULT_REDIS_PORT: u16 = 6379; diff --git a/core/core/src/services/redis/config.rs b/core/services/redis/src/config.rs similarity index 91% rename from core/core/src/services/redis/config.rs rename to core/services/redis/src/config.rs index 7df08e87c6d1..e5aab2a75ed1 100644 --- a/core/core/src/services/redis/config.rs +++ b/core/services/redis/src/config.rs @@ -17,12 +17,13 @@ use std::fmt::Debug; +use opendal_core::raw::*; +use opendal_core::*; use serde::Deserialize; use serde::Serialize; use super::REDIS_SCHEME; use super::backend::RedisBuilder; -use crate::raw::*; /// Config for Redis services support. #[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] @@ -74,18 +75,18 @@ impl Debug for RedisConfig { } } -impl crate::Configurator for RedisConfig { +impl Configurator for RedisConfig { type Builder = RedisBuilder; - fn from_uri(uri: &crate::types::OperatorUri) -> crate::Result { + fn from_uri(uri: &OperatorUri) -> Result { let mut map = uri.options().clone(); if let Some(authority) = uri.authority() { map.entry("endpoint".to_string()) .or_insert_with(|| format!("redis://{authority}")); } else if !map.contains_key("endpoint") && !map.contains_key("cluster_endpoints") { - return Err(crate::Error::new( - crate::ErrorKind::ConfigInvalid, + return Err(Error::new( + ErrorKind::ConfigInvalid, "endpoint or cluster_endpoints is required", ) .with_context("service", REDIS_SCHEME)); @@ -126,35 +127,33 @@ impl crate::Configurator for RedisConfig { #[cfg(test)] mod tests { use super::*; - use crate::Configurator; - use crate::types::OperatorUri; #[test] - fn from_uri_sets_endpoint_db_and_root() { + fn from_uri_sets_endpoint_db_and_root() -> Result<()> { let uri = OperatorUri::new( "redis://localhost:6379/2/cache", Vec::<(String, String)>::new(), - ) - .unwrap(); + )?; - let cfg = RedisConfig::from_uri(&uri).unwrap(); + let cfg = RedisConfig::from_uri(&uri)?; assert_eq!(cfg.endpoint.as_deref(), Some("redis://localhost:6379")); assert_eq!(cfg.db, 2); assert_eq!(cfg.root.as_deref(), Some("cache")); + Ok(()) } #[test] - fn from_uri_treats_non_numeric_path_as_root() { + fn from_uri_treats_non_numeric_path_as_root() -> Result<()> { let uri = OperatorUri::new( "redis://localhost:6379/app/data", Vec::<(String, String)>::new(), - ) - .unwrap(); + )?; - let cfg = RedisConfig::from_uri(&uri).unwrap(); + let cfg = RedisConfig::from_uri(&uri)?; assert_eq!(cfg.endpoint.as_deref(), Some("redis://localhost:6379")); assert_eq!(cfg.db, 0); assert_eq!(cfg.root.as_deref(), Some("app/data")); + Ok(()) } #[test] diff --git a/core/core/src/services/redis/core.rs b/core/services/redis/src/core.rs similarity index 99% rename from core/core/src/services/redis/core.rs rename to core/services/redis/src/core.rs index 08fedc5d3ed4..1f3574a563e7 100644 --- a/core/core/src/services/redis/core.rs +++ b/core/services/redis/src/core.rs @@ -32,8 +32,8 @@ use redis::aio::ConnectionManager; use redis::cluster::ClusterClient; use redis::cluster_async::ClusterConnection; -use crate::raw::*; -use crate::*; +use opendal_core::raw::*; +use opendal_core::*; #[derive(Clone)] pub enum RedisConnection { diff --git a/core/core/src/services/redis/delete.rs b/core/services/redis/src/delete.rs similarity index 87% rename from core/core/src/services/redis/delete.rs rename to core/services/redis/src/delete.rs index 2a0f066e05dd..e481ddfc35f4 100644 --- a/core/core/src/services/redis/delete.rs +++ b/core/services/redis/src/delete.rs @@ -15,18 +15,20 @@ // specific language governing permissions and limitations // under the License. +use std::sync::Arc; + +use opendal_core::raw::*; +use opendal_core::*; + use super::core::RedisCore; -use crate::raw::oio; -use crate::raw::*; -use crate::*; pub struct RedisDeleter { - core: std::sync::Arc, + core: Arc, root: String, } impl RedisDeleter { - pub fn new(core: std::sync::Arc, root: String) -> Self { + pub fn new(core: Arc, root: String) -> Self { Self { core, root } } } diff --git a/core/core/src/services/redis/docs.md b/core/services/redis/src/docs.md similarity index 94% rename from core/core/src/services/redis/docs.md rename to core/services/redis/src/docs.md index 2f7e930d066a..ff7fed8f303d 100644 --- a/core/core/src/services/redis/docs.md +++ b/core/services/redis/src/docs.md @@ -28,9 +28,9 @@ You can refer to [`RedisBuilder`]'s docs for more information ### Via Builder ```rust,no_run -use anyhow::Result; -use opendal_core::services::Redis; use opendal_core::Operator; +use opendal_core::Result; +use opendal_service_redis::Redis; #[tokio::main] async fn main() -> Result<()> { diff --git a/core/core/src/services/redis/mod.rs b/core/services/redis/src/lib.rs similarity index 84% rename from core/core/src/services/redis/mod.rs rename to core/services/redis/src/lib.rs index 6d0563bcc02d..a2f99e58b01e 100644 --- a/core/core/src/services/redis/mod.rs +++ b/core/services/redis/src/lib.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -/// Default scheme for redis service. -pub const REDIS_SCHEME: &str = "redis"; +//! Redis service implementation for Apache OpenDAL. -use crate::types::DEFAULT_OPERATOR_REGISTRY; +#![cfg_attr(docsrs, feature(doc_cfg))] +#![deny(missing_docs)] mod backend; mod config; @@ -29,7 +29,10 @@ mod writer; pub use backend::RedisBuilder as Redis; pub use config::RedisConfig; +/// Default scheme for redis service. +pub const REDIS_SCHEME: &str = "redis"; + #[ctor::ctor] fn register_redis_service() { - DEFAULT_OPERATOR_REGISTRY.register::(REDIS_SCHEME); + opendal_core::DEFAULT_OPERATOR_REGISTRY.register::(REDIS_SCHEME); } diff --git a/core/core/src/services/redis/writer.rs b/core/services/redis/src/writer.rs similarity index 91% rename from core/core/src/services/redis/writer.rs rename to core/services/redis/src/writer.rs index a2ad22dd1967..b96b50378fb7 100644 --- a/core/core/src/services/redis/writer.rs +++ b/core/services/redis/src/writer.rs @@ -15,18 +15,21 @@ // specific language governing permissions and limitations // under the License. +use std::sync::Arc; + +use opendal_core::raw::*; +use opendal_core::*; + use super::core::RedisCore; -use crate::raw::oio; -use crate::*; pub struct RedisWriter { - core: std::sync::Arc, + core: Arc, path: String, buffer: oio::QueueBuf, } impl RedisWriter { - pub fn new(core: std::sync::Arc, path: String) -> Self { + pub fn new(core: Arc, path: String) -> Self { Self { core, path, diff --git a/core/src/lib.rs b/core/src/lib.rs index 502185981752..f45746d41068 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -120,6 +120,8 @@ pub mod services { pub use opendal_service_postgresql::*; #[cfg(feature = "services-redb")] pub use opendal_service_redb::*; + #[cfg(feature = "services-redis")] + pub use opendal_service_redis::*; #[cfg(feature = "services-rocksdb")] pub use opendal_service_rocksdb::*; #[cfg(feature = "services-s3")] @@ -140,6 +142,8 @@ pub mod services { pub use opendal_service_tikv::*; #[cfg(feature = "services-upyun")] pub use opendal_service_upyun::*; + #[cfg(feature = "services-vercel-artifacts")] + pub use opendal_service_vercel_artifacts::*; #[cfg(feature = "services-vercel-blob")] pub use opendal_service_vercel_blob::*; #[cfg(feature = "services-webdav")] From 0d13b43376ff6d3ed8712a98c50c17ddbeaea24e Mon Sep 17 00:00:00 2001 From: koushiro Date: Tue, 23 Dec 2025 15:21:48 +0800 Subject: [PATCH 2/2] fix license check and redis service features --- core/Cargo.toml | 7 +++++-- core/services/redis/Cargo.toml | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/core/Cargo.toml b/core/Cargo.toml index 5a6d1cc3fb75..f3f58c972ab2 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -160,8 +160,11 @@ services-pcloud = ["dep:opendal-service-pcloud"] services-persy = ["dep:opendal-service-persy"] services-postgresql = ["dep:opendal-service-postgresql"] services-redb = ["dep:opendal-service-redb"] -services-redis = ["opendal-service-redis?/rustls"] -services-redis-native-tls = ["opendal-service-redis?/native-tls"] +services-redis = ["dep:opendal-service-redis", "opendal-service-redis?/rustls"] +services-redis-native-tls = [ + "dep:opendal-service-redis", + "opendal-service-redis?/native-tls", +] services-rocksdb = ["dep:opendal-service-rocksdb"] services-s3 = ["dep:opendal-service-s3"] services-seafile = ["dep:opendal-service-seafile"] diff --git a/core/services/redis/Cargo.toml b/core/services/redis/Cargo.toml index f8587a98ebe2..e28c9ba6748c 100644 --- a/core/services/redis/Cargo.toml +++ b/core/services/redis/Cargo.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [package] description = "Apache OpenDAL Redis service implementation" name = "opendal-service-redis"