feat: add method to parse image from fsuri, like btcfs, ipfs and ckbfs

Cargo.toml

@@ -6,28 +6,37 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+base64 = "0.22.1"
 ckb-sdk = "3.2.0"
 ckb-types = "0.116.1"
 ckb-jsonrpc-types = "0.116.1"
 ckb-hash = "0.116.1"
 thiserror = "1.0"
 serde_json = "1.0"
 hex = "0.4.3"
-reqwest = { version = "0.12.4", features = ["json"] }
+svg = "0.15"
+ttf-parser = "0.24"
+lyon = "1.0"
+reqwest = { version = "0.12.4", features = ["json", "rustls-tls", "trust-dns"] }
 jsonrpc-core = "18.0"
 serde = { version = "1.0", features = ["serde_derive"] }
 futures = "0.3"
-lazy_static = { version = "1.4" }
+lazy_static = "1.4"
+lazy-regex = "3.1.0"
+chrono = { version = "0.4", features = ["serde"] }
 ckb-vm = { version = "0.24", features = ["asm"] }
+axum = { version = "0.7", features = ["macros"] }
 
 spore-types = { git = "https://github.com/sporeprotocol/spore-contract", rev = "81315ca" }
 
-jsonrpsee = { version = "0.22.3", features = ["server", "macros"], optional = true }
+jsonrpsee = { version = "0.26", features = ["server", "macros"], optional = true }
+tower-http = { version = "0.6", features = ["cors"], optional = true }
+tower = { version = "0.5", optional = true }
 toml = { version = "0.8.2", optional = true }
 tokio = { version = "1.37", features = ["rt", "signal"], optional = true }
 tracing-subscriber = { version = "0.3.18", features = ["fmt", "env-filter"], optional = true }
 
 [features]
 default = ["standalone_server", "render_debug"]
-standalone_server = ["jsonrpsee", "toml", "tokio", "tracing-subscriber"]
+standalone_server = ["jsonrpsee", "toml", "tokio", "tracing-subscriber", "tower-http", "tower"]
 render_debug = []

README.md

@@ -40,13 +40,57 @@ $ RUST_LOG=dob_decoder_server=debug cargo run
 
 And then, try it out:
 
+**Get protocol versions:**
+
+```bash
+$ curl -H 'content-type: application/json' -d '{
+    "id": 1,
+    "jsonrpc": "2.0",
+    "method": "dob_protocol_version",
+    "params": []
+}' http://localhost:8090
+```
+
+**Decode a spore ID:**
+
 ```bash
 $ echo '{
     "id": 2,
     "jsonrpc": "2.0",
     "method": "dob_decode",
     "params": [
-        "<spore_id in hex format without 0x prefix>"
+        "4f7fb83a65dae9b95c21e55d5776a84f17bb6377681befeedb20a077ce1d8aad"
+    ]
+}' \
+| curl -H 'content-type: application/json' -d @- \
+http://localhost:8090
+```
+
+**Decode and extract SVG from another example spore ID on mainnet:**
+
+```bash
+$ echo '{
+    "id": 3,
+    "jsonrpc": "2.0",
+    "method": "dob_decode_svg",
+    "params": [
+        "0x577bf0de0dcffe2811fa827480a700bc800c8e1e9606615b1484baeea2cba830"
+    ]
+}' \
+| curl -H 'content-type: application/json' -d @- \
+http://localhost:8090
+```
+
+**Extract image from a btcfs or ipfs path:**
+
+```bash
+$ echo '{
+    "id": 4,
+    "jsonrpc": "2.0",
+    "method": "dob_extract_image_from_fsuri",
+    "params": [
+        "btcfs://5895004e95c8a4b80f05f5314d310067a703134515d82effc2ec6eba0dda3fc9i0",
+        "base64"
     ]
 }' \
 | curl -H 'content-type: application/json' -d @- \

settings.mainnet.toml

@@ -7,6 +7,9 @@ protocol_versions = [
 # connect to the RPC of CKB node
 ckb_rpc = "https://mainnet.ckb.dev/"
 
+# connect to the image fetcher service
+image_fetcher_url = { btcfs = "https://mempool.space/api/tx/", ipfs = "https://ipfs.io/ipfs/" }
+
 # address that rpc server running at in case of standalone server mode
 rpc_server_address = "0.0.0.0:8090"
 
@@ -19,6 +22,9 @@ dobs_cache_directory = "cache/dobs"
 # expiration time indicator for cleaning whole dobs cache, zero means never clean
 dobs_cache_expiration_sec = 300
 
+# expiration time in minutes dedicated for type_id and type_script decoder cache, zero means always update
+decoders_cache_expiration_minutes = 1440
+
 # all deployed on-chain Spore contracts binary hash (order from new to old)
 # refer to: https://github.com/sporeprotocol/spore-contract/blob/master/docs/VERSIONS.md
 [[available_spores]]

settings.toml

@@ -7,18 +7,27 @@ protocol_versions = [
 # connect to the RPC of CKB node
 ckb_rpc = "https://testnet.ckbapp.dev/"
 
+# connect to the image fetcher service
+image_fetcher_url = { btcfs = "https://mempool.space/api/tx/", ipfs = "https://ipfs.io/ipfs/" }
+
 # address that rpc server running at in case of standalone server mode
 rpc_server_address = "0.0.0.0:8090"
 
+# address that restful api server running at
+restful_server_address = "0.0.0.0:8091"
+
 # directory that stores decoders on hard-disk, including on-chain and off-chain binary files
-decoders_cache_directory = "cache/decoders"
+decoders_cache_directory = "cache/decoders/testnet"
 
 # directory that stores DOBs rendering results on hard-disk
 dobs_cache_directory = "cache/dobs"
 
 # expiration time indicator for cleaning whole dobs cache, zero means never clean
 dobs_cache_expiration_sec = 300
 
+# expiration time in minutes dedicated for type_id and type_script decoder cache, zero means always update
+decoders_cache_expiration_minutes = 1440
+
 # all deployed on-chain Spore contracts binary hash (order from new to old)
 # refer to: https://github.com/sporeprotocol/spore-contract/blob/master/docs/VERSIONS.md
 [[available_spores]]

src/client.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use std::future::Future;
 use std::pin::Pin;
 use std::sync::atomic::{AtomicU64, Ordering};
@@ -9,7 +10,9 @@ use ckb_jsonrpc_types::{
 use ckb_sdk::rpc::ckb_indexer::{Cell, Order, Pagination, SearchKey, Tx};
 use ckb_types::H256;
 use jsonrpc_core::futures::FutureExt;
+use lazy_regex::{regex, regex_replace_all};
 use reqwest::{Client, Url};
+use serde_json::Value;
 
 use crate::types::Error;
 
@@ -150,3 +153,166 @@ impl RpcClient {
         .boxed()
     }
 }
+
+#[derive(Clone)]
+pub struct ImageFetchClient {
+    base_url: HashMap<String, Url>,
+}
+
+impl ImageFetchClient {
+    pub fn new(base_url: &HashMap<String, Url>) -> Self {
+        Self {
+            base_url: base_url.clone(),
+        }
+    }
+
+    pub async fn fetch_images(&self, images_uri: &[String]) -> Result<Vec<Vec<u8>>, Error> {
+        let mut requests = vec![];
+        for uri in images_uri {
+            match uri.try_into()? {
+                URI::BTCFS(tx_hash, index) => {
+                    let url = self
+                        .base_url
+                        .get("btcfs")
+                        .ok_or(Error::FsuriNotFoundInConfig)?
+                        .join(&tx_hash)
+                        .expect("image url");
+                    requests.push(parse_image_from_btcfs(url, index).boxed());
+                }
+                URI::IPFS(cid) => {
+                    let url = self
+                        .base_url
+                        .get("ipfs")
+                        .ok_or(Error::FsuriNotFoundInConfig)?
+                        .join(&cid)
+                        .expect("image url");
+                    requests.push(
+                        async move {
+                            let image = reqwest::get(url.clone())
+                                .await
+                                .map_err(|e| Error::FetchFromIpfsError(e.to_string()))?
+                                .bytes()
+                                .await
+                                .map_err(|e| Error::FetchFromIpfsError(e.to_string()))?
+                                .to_vec();
+                            Ok(image)
+                        }
+                        .boxed(),
+                    );
+                }
+            }
+        }
+        let mut images = vec![];
+        let responses = futures::future::join_all(requests).await;
+        for response in responses {
+            images.push(response?);
+        }
+        Ok(images)
+    }
+}
+
+#[allow(clippy::upper_case_acronyms)]
+enum URI {
+    BTCFS(String, usize),
+    IPFS(String),
+}
+
+impl TryFrom<&String> for URI {
+    type Error = Error;
+
+    fn try_from(uri: &String) -> Result<Self, Error> {
+        if let Some(body) = uri.strip_prefix("btcfs://") {
+            let parts: Vec<&str> = body.split('i').collect::<Vec<_>>();
+            if parts.len() != 2 {
+                return Err(Error::InvalidOnchainFsuriFormat);
+            }
+            let tx_hash = parts[0].to_string();
+            let index = parts[1]
+                .parse()
+                .map_err(|_| Error::InvalidOnchainFsuriFormat)?;
+            Ok(URI::BTCFS(tx_hash, index))
+        } else if let Some(body) = uri.strip_prefix("ipfs://") {
+            let hash = body.to_string();
+            Ok(URI::IPFS(hash))
+        } else {
+            Err(Error::InvalidOnchainFsuriFormat)
+        }
+    }
+}
+
+async fn parse_image_from_btcfs(url: Url, index: usize) -> Result<Vec<u8>, Error> {
+    // parse btc transaction
+    let btc_tx = reqwest::get(url)
+        .await
+        .map_err(|e| Error::FetchFromBtcNodeError(e.to_string()))?
+        .json::<Value>()
+        .await
+        .map_err(|e| Error::FetchFromBtcNodeError(e.to_string()))?;
+    let vin = btc_tx
+        .get("vin")
+        .ok_or(Error::InvalidBtcTransactionFormat(
+            "vin not found".to_string(),
+        ))?
+        .as_array()
+        .ok_or(Error::InvalidBtcTransactionFormat(
+            "vin not an array".to_string(),
+        ))?
+        .first()
+        .ok_or(Error::InvalidBtcTransactionFormat(
+            "vin is empty".to_string(),
+        ))?;
+    let witness = vin
+        .get("inner_witnessscript_asm")
+        .ok_or(Error::InvalidBtcTransactionFormat(
+            "inner_witnessscript_asm not found".to_string(),
+        ))?
+        .as_str()
+        .ok_or(Error::InvalidBtcTransactionFormat(
+            "inner_witnessscript_asm not a string".to_string(),
+        ))?
+        .to_owned();
+
+    // parse inscription body
+    let mut images = vec![];
+    let mut witness_view = witness.as_str();
+
+    // flexible header pattern, e.g. OP_IF OP_PUSHBYTES_3 444f42 OP_PUSHBYTES_1 01 OP_PUSHBYTES_9 696d6167652f706e67 OP_0 OP_PUSHDATA2
+    // note: arbitrary part is the specification of image type
+    let header_pattern = regex!(
+        r#"OP_IF\s+OP_PUSHBYTES_3\s+444f42\s+OP_PUSHBYTES_1\s+01\s+OP_PUSHBYTES_(\d+)\s+([0-9a-fA-F]+)\s+OP_0\s+OP_PUSHDATA2\s+"#
+    );
+
+    while let (Some(start), Some(end)) = (witness_view.find("OP_IF"), witness_view.find("OP_ENDIF"))
+    {
+        if start >= end {
+            return Err(Error::InvalidInscriptionFormat(
+                "bad start and end position".to_string(),
+            ));
+        }
+        let inscription = &witness_view[start..end + "OP_ENDIF".len()];
+
+        if let Some(captures) = header_pattern.captures(inscription) {
+            let matched_header = &captures[0];
+            let base_removed = inscription.replace(matched_header, "");
+            let hexed = regex_replace_all!(r#"\s?OP\_\w+\s?"#, &base_removed, "");
+            let image = hex::decode(hexed.as_bytes())
+                .map_err(|_| Error::InvalidInscriptionContentHexFormat)?;
+            images.push(image);
+        } else {
+            return Err(Error::InvalidInscriptionFormat(
+                "HEADER pattern not found".to_string(),
+            ));
+        }
+
+        witness_view = &witness_view[end + "OP_ENDIF".len()..];
+    }
+    if images.is_empty() {
+        return Err(Error::EmptyInscriptionContent);
+    }
+
+    let image = images
+        .get(index)
+        .cloned()
+        .ok_or(Error::ExceededInscriptionIndex)?;
+    Ok(image)
+}

src/decoder/helpers.rs

@@ -32,6 +32,26 @@ fn build_type_script_search_option(type_script: Script) -> CellQueryOptions {
     CellQueryOptions::new_type(type_script)
 }
 
+fn file_older_than_minutes(file_path: &PathBuf, minutes: u64) -> bool {
+    if minutes == 0 {
+        return true;
+    }
+    let metadata = match std::fs::metadata(file_path) {
+        Ok(m) => m,
+        Err(_) => return true, // File doesn't exist or we can't access it, so it's "old".
+    };
+
+    let file_time = match metadata.modified().or_else(|_| metadata.created()) {
+        Ok(time) => time,
+        Err(_) => return true, // Can't get a timestamp, assume it's old.
+    };
+
+    match file_time.elapsed() {
+        Ok(elapsed) => elapsed.as_secs() >= minutes * 60,
+        Err(_) => true, // System clock is earlier than file time, assume it's old to be safe.
+    }
+}
+
 fn build_batch_search_options(
     type_args: &[u8; 32],
     available_script_ids: &[ScriptId],
@@ -80,7 +100,9 @@ pub fn decode_spore_content(content: &[u8]) -> Result<(Value, String), Error> {
         return Ok((serde_json::Value::String(dna.clone()), dna));
     }
 
-    let value: Value = serde_json::from_slice(content).map_err(|_| Error::DOBContentUnexpected)?;
+    // if content is not a valid JSON, treat it as a DNA string
+    let value: Value = serde_json::from_slice(content)
+        .unwrap_or(Value::String(String::from_utf8_lossy(content).to_string()));
     let dna = match &value {
         serde_json::Value::String(_) => &value,
         serde_json::Value::Array(array) => array.first().ok_or(Error::DOBContentUnexpected)?,
@@ -267,7 +289,7 @@ pub async fn parse_decoder_path(
         DecoderLocationType::TypeId => {
             let hash = decoder.hash.as_ref().ok_or(Error::DecoderHashNotFound)?;
             decoder_path.push(format!("type_id_{}.bin", hex::encode(hash)));
-            if !decoder_path.exists() {
+            if file_older_than_minutes(&decoder_path, settings.decoders_cache_expiration_minutes) {
                 let decoder_search_option = build_type_id_search_option(hash.clone().into());
                 let decoder_binary = fetch_decoder_binary(rpc, decoder_search_option).await?;
                 std::fs::write(decoder_path.clone(), decoder_binary)
@@ -284,7 +306,7 @@ pub async fn parse_decoder_path(
                 "type_script_{}.bin",
                 hex::encode(script.calc_script_hash().raw_data())
             ));
-            if !decoder_path.exists() {
+            if file_older_than_minutes(&decoder_path, settings.decoders_cache_expiration_minutes) {
                 let decoder_search_option = build_type_script_search_option(script);
                 let decoder_binary = fetch_decoder_binary(rpc, decoder_search_option).await?;
                 std::fs::write(decoder_path.clone(), decoder_binary)

src/decoder/mod.rs

@@ -11,6 +11,7 @@ use crate::{
 pub(crate) mod helpers;
 use helpers::*;
 
+#[derive(Clone)]
 pub struct DOBDecoder {
     rpc: RpcClient,
     settings: Settings,