Kyle/public config access pyo3 object store

obstore/Cargo.toml

@@ -35,7 +35,8 @@ pyo3-arrow = "0.16"
 pyo3-async-runtimes = { workspace = true, features = ["tokio-runtime"] }
 pyo3-bytes = "0.6"
 pyo3-file = { workspace = true }
-pyo3-object_store = { path = "../pyo3-object_store" }
+# pyo3-object_store = { path = "../pyo3-object_store" }
+pyo3-object_store = { git = "https://github.com/developmentseed/obstore", rev = "20010a52a1eaa95e5873c181f08a1cf768a97df4" }
 tokio = { workspace = true, features = [
     "macros",
     "rt",

pyo3-object_store/Cargo.toml

@@ -36,8 +36,8 @@ object_store = { version = "0.13.0", features = [
 ] }
 # This is already an object_store dependency
 percent-encoding = "2.1"
-pyo3 = { version = "0.28", features = ["chrono", "indexmap"] }
-pyo3-async-runtimes = { version = "0.28", features = ["tokio-runtime"] }
+pyo3 = { version = "0.27", features = ["chrono", "indexmap"] }
+pyo3-async-runtimes = { version = "0.27", features = ["tokio-runtime"] }
 serde = "1"
 thiserror = "1"
 tokio = { version = "1.40", features = ["rt-multi-thread"] }

pyo3-object_store/src/aws/credentials.rs

@@ -53,6 +53,7 @@ impl<'py> FromPyObject<'_, 'py> for PyAwsCredential {
     }
 }
 
+/// A Python-facing wrapper around a user-provided callback for AWS credential management.
 // TODO: don't use a cache for static credentials where `expires_at` is `None`
 // (so you don't need to access a mutex)
 #[derive(Debug)]

pyo3-object_store/src/aws/mod.rs

@@ -1,4 +1,7 @@
+//! AWS S3 object store binding.
+
 mod credentials;
 mod store;
 
-pub use store::PyS3Store;
+pub use credentials::PyAWSCredentialProvider;
+pub use store::{PyAmazonS3Config, PyAmazonS3ConfigKey, PyS3Store};

pyo3-object_store/src/aws/store.rs

@@ -20,7 +20,7 @@ use crate::retry::PyRetryConfig;
 use crate::PyUrl;
 
 #[derive(Debug, Clone, PartialEq)]
-struct S3Config {
+pub struct S3Config {
     prefix: Option<PyPath>,
     config: PyAmazonS3Config,
     client_options: Option<PyClientOptions>,
@@ -29,14 +29,42 @@ struct S3Config {
 }
 
 impl S3Config {
-    fn bucket(&self) -> &str {
+    /// Access the bucket name for this config.
+    pub fn bucket(&self) -> &str {
         self.config
             .0
             .get(&PyAmazonS3ConfigKey(AmazonS3ConfigKey::Bucket))
             .expect("bucket should always exist in the config")
             .as_ref()
     }
 
+    /// Access the prefix for this config, if it exists.
+    pub fn prefix(&self) -> Option<&PyPath> {
+        self.prefix.as_ref()
+    }
+
+    /// Access the config key-value pairs for this config.
+    ///
+    /// Note that the bucket **is included** in the returned config.
+    pub fn config(&self) -> &PyAmazonS3Config {
+        &self.config
+    }
+
+    /// Access the client options for this config, if they exist.
+    pub fn client_options(&self) -> Option<&PyClientOptions> {
+        self.client_options.as_ref()
+    }
+
+    /// Access the retry config for this config, if it exists.
+    pub fn retry_config(&self) -> Option<&PyRetryConfig> {
+        self.retry_config.as_ref()
+    }
+
+    /// Access the credential provider for this config, if it exists.
+    pub fn credential_provider(&self) -> Option<&PyAWSCredentialProvider> {
+        self.credential_provider.as_ref()
+    }
+
     fn __getnewargs_ex__<'py>(&'py self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
         let args = PyTuple::empty(py).into_bound_py_any(py)?;
         let kwargs = PyDict::new(py);
@@ -79,6 +107,11 @@ impl PyS3Store {
     pub fn into_inner(self) -> Arc<MaybePrefixedStore<AmazonS3>> {
         self.store
     }
+
+    /// Access the config for this store.
+    pub fn config(&self) -> &S3Config {
+        &self.config
+    }
 }
 
 #[pymethods]
@@ -201,8 +234,9 @@ impl PyS3Store {
         self.config.prefix.as_ref()
     }
 
+    #[pyo3(name = "config")]
     #[getter]
-    fn config(&self) -> &PyAmazonS3Config {
+    fn py_config(&self) -> &PyAmazonS3Config {
         &self.config.config
     }
 
@@ -222,6 +256,7 @@ impl PyS3Store {
     }
 }
 
+/// A Python-facing wrapper around a config key for S3 configuration.
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct PyAmazonS3ConfigKey(AmazonS3ConfigKey);
 
@@ -235,6 +270,12 @@ impl<'py> FromPyObject<'_, 'py> for PyAmazonS3ConfigKey {
     }
 }
 
+impl AsRef<AmazonS3ConfigKey> for PyAmazonS3ConfigKey {
+    fn as_ref(&self) -> &AmazonS3ConfigKey {
+        &self.0
+    }
+}
+
 impl AsRef<str> for PyAmazonS3ConfigKey {
     fn as_ref(&self) -> &str {
         self.0.as_ref()
@@ -278,9 +319,16 @@ impl From<PyAmazonS3ConfigKey> for AmazonS3ConfigKey {
     }
 }
 
+/// A Python-facing wrapper around a set of S3 configuration key-value pairs.
 #[derive(Clone, Debug, Default, PartialEq, Eq, IntoPyObject, IntoPyObjectRef)]
 pub struct PyAmazonS3Config(HashMap<PyAmazonS3ConfigKey, PyConfigValue>);
 
+impl AsRef<HashMap<PyAmazonS3ConfigKey, PyConfigValue>> for PyAmazonS3Config {
+    fn as_ref(&self) -> &HashMap<PyAmazonS3ConfigKey, PyConfigValue> {
+        &self.0
+    }
+}
+
 // Note: we manually impl FromPyObject instead of deriving it so that we can raise an
 // UnknownConfigurationKeyError instead of a `TypeError` on invalid config keys.
 //

pyo3-object_store/src/azure/credentials.rs

@@ -17,7 +17,8 @@ use crate::credentials::{is_awaitable, TemporaryToken, TokenCache};
 use crate::path::PyPath;
 use crate::PyObjectStoreError;
 
-struct PyAzureAccessKey {
+/// A wrapper around an [AzureAccessKey]
+pub struct PyAzureAccessKey {
     access_key: AzureAccessKey,
     expires_at: Option<DateTime<Utc>>,
 }
@@ -40,7 +41,8 @@ impl<'py> FromPyObject<'_, 'py> for PyAzureAccessKey {
     }
 }
 
-struct PyAzureSASToken {
+/// A wrapper around a SAS token, which is a list of key-value pairs, and an optional expiry timestamp.
+pub struct PyAzureSASToken {
     sas_token: Vec<(String, String)>,
     expires_at: Option<DateTime<Utc>>,
 }
@@ -70,7 +72,8 @@ impl<'py> FromPyObject<'_, 'py> for PyAzureSASToken {
     }
 }
 
-struct PyBearerToken {
+/// A wrapper around a bearer token
+pub struct PyBearerToken {
     token: String,
     expires_at: Option<DateTime<Utc>>,
 }
@@ -86,10 +89,16 @@ impl<'py> FromPyObject<'_, 'py> for PyBearerToken {
     }
 }
 
+/// A Python-facing enum wrapper around different Azure credential types.
 #[derive(FromPyObject)]
-enum PyAzureCredential {
+pub enum PyAzureCredential {
+    /// An access key credential
     AccessKey(PyAzureAccessKey),
+
+    /// A SAS token credential
     SASToken(PyAzureSASToken),
+
+    /// A bearer token credential
     BearerToken(PyBearerToken),
 }
 
@@ -141,6 +150,7 @@ fn split_sas(sas: &str) -> Result<Vec<(String, String)>, object_store::Error> {
     Ok(pairs)
 }
 
+/// A Python-facing wrapper around a user-provided credential provider callback
 #[derive(Debug)]
 pub struct PyAzureCredentialProvider {
     /// The provided user callback to manage credential refresh

pyo3-object_store/src/azure/mod.rs

@@ -1,5 +1,10 @@
+//! Azure Object Store Python bindings.
+
 mod credentials;
 mod error;
 mod store;
 
-pub use store::PyAzureStore;
+pub use credentials::{
+    PyAzureAccessKey, PyAzureCredential, PyAzureCredentialProvider, PyAzureSASToken, PyBearerToken,
+};
+pub use store::{PyAzureConfig, PyAzureConfigKey, PyAzureStore};

pyo3-object_store/src/azure/store.rs

@@ -18,7 +18,7 @@ use crate::retry::PyRetryConfig;
 use crate::{MaybePrefixedStore, PyUrl};
 
 #[derive(Debug, Clone, PartialEq)]
-struct AzureConfig {
+pub struct AzureConfig {
     prefix: Option<PyPath>,
     config: PyAzureConfig,
     client_options: Option<PyClientOptions>,
@@ -43,6 +43,33 @@ impl AzureConfig {
             .as_ref()
     }
 
+    /// Access the prefix for this config, if it exists.
+    pub fn prefix(&self) -> Option<&PyPath> {
+        self.prefix.as_ref()
+    }
+
+    /// Access the config key-value pairs for this config.
+    ///
+    /// Note that the account name and container name **are included** in the returned config.
+    pub fn config(&self) -> &PyAzureConfig {
+        &self.config
+    }
+
+    /// Access the client options for this config, if they exist.
+    pub fn client_options(&self) -> Option<&PyClientOptions> {
+        self.client_options.as_ref()
+    }
+
+    /// Access the retry config for this config, if it exists.
+    pub fn retry_config(&self) -> Option<&PyRetryConfig> {
+        self.retry_config.as_ref()
+    }
+
+    /// Access the credential provider for this config, if it exists.
+    pub fn credential_provider(&self) -> Option<&PyAzureCredentialProvider> {
+        self.credential_provider.as_ref()
+    }
+
     fn __getnewargs_ex__<'py>(&'py self, py: Python<'py>) -> PyResult<Bound<'py, PyTuple>> {
         let args = PyTuple::empty(py).into_bound_py_any(py)?;
         let kwargs = PyDict::new(py);
@@ -85,6 +112,11 @@ impl PyAzureStore {
     pub fn into_inner(self) -> Arc<MaybePrefixedStore<MicrosoftAzure>> {
         self.store
     }
+
+    /// Access the config for this store.
+    pub fn config(&self) -> &AzureConfig {
+        &self.config
+    }
 }
 
 #[pymethods]
@@ -223,8 +255,9 @@ impl PyAzureStore {
         self.config.prefix.as_ref()
     }
 
+    #[pyo3(name = "config")]
     #[getter]
-    fn config(&self) -> &PyAzureConfig {
+    fn py_config(&self) -> &PyAzureConfig {
         &self.config.config
     }
 
@@ -244,6 +277,7 @@ impl PyAzureStore {
     }
 }
 
+/// A Python-facing wrapper around a config key for Azure configuration.
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub struct PyAzureConfigKey(AzureConfigKey);
 
@@ -304,6 +338,7 @@ impl From<PyAzureConfigKey> for AzureConfigKey {
     }
 }
 
+/// A Python-facing wrapper around a config for Azure configuration.
 #[derive(Clone, Debug, Default, PartialEq, Eq, IntoPyObject, IntoPyObjectRef)]
 pub struct PyAzureConfig(HashMap<PyAzureConfigKey, PyConfigValue>);
 

pyo3-object_store/src/gcp/credentials.rs

@@ -43,6 +43,7 @@ impl<'py> FromPyObject<'_, 'py> for PyGcpCredential {
     }
 }
 
+/// A Python-facing wrapper around a user-provided callback for GCP credential management.
 // TODO: don't use a cache for static credentials where `expires_at` is `None`
 // (so you don't need to access a mutex)
 #[derive(Debug)]

pyo3-object_store/src/gcp/mod.rs

@@ -1,4 +1,7 @@
+//! Google Cloud Storage Object Store Python bindings.
+
 mod credentials;
 mod store;
 
-pub use store::PyGCSStore;
+pub use credentials::PyGcpCredentialProvider;
+pub use store::{PyGCSStore, PyGoogleConfig, PyGoogleConfigKey};