add build-time ETag to /-/static/* responses & return "304 Not Modified"

Author: syphar

Cargo.lock

@@ -102,6 +102,7 @@ fn-error-context = "0.2.0"
 # Templating
 askama = "0.14.0"
 walkdir = "2"
+phf = "0.13.1"
 
 # Date and Time utilities
 chrono = { version = "0.4.11", default-features = false, features = ["clock", "serde"] }
@@ -110,6 +111,7 @@ chrono = { version = "0.4.11", default-features = false, features = ["clock", "s
 thread_local = "1.1.3"
 constant_time_eq = "0.4.2"
 fastly-api = "12.0.0"
+md5 = "0.8.0"
 
 [dev-dependencies]
 criterion = "0.7.0"
@@ -132,8 +134,10 @@ debug = "line-tables-only"
 
 [build-dependencies]
 time = "0.3"
+md5 = "0.8.0"
 gix = { version = "0.74.0", default-features = false }
 string_cache_codegen = "0.6.1"
+phf_codegen = "0.13"
 walkdir = "2"
 anyhow = { version = "1.0.42", features = ["backtrace"] }
 grass = { version = "0.13.1", default-features = false }

Cargo.toml

@@ -1,5 +1,5 @@
 use anyhow::{Context as _, Error, Result};
-use std::{env, path::Path};
+use std::{env, fs::File, io::Write as _, path::Path};
 
 mod tracked {
     use std::{
@@ -68,13 +68,27 @@ mod tracked {
     }
 }
 
+type ETagMap<'a> = phf_codegen::Map<'a, String>;
+
 fn main() -> Result<()> {
     let out_dir = env::var("OUT_DIR").context("missing OUT_DIR")?;
     let out_dir = Path::new(&out_dir);
     read_git_version()?;
-    compile_sass(out_dir)?;
+
+    let mut etag_map: ETagMap = ETagMap::new();
+
+    compile_sass(out_dir, &mut etag_map)?;
     write_known_targets(out_dir)?;
     compile_syntax(out_dir).context("could not compile syntax files")?;
+    calculate_static_etags(&mut etag_map)?;
+
+    let mut etag_file = File::create(out_dir.join("static_etag_map.rs"))?;
+    writeln!(
+        &mut etag_file,
+        "pub static STATIC_ETAG_MAP: ::phf::Map<&'static str, &'static str> = {};",
+        etag_map.build()
+    )?;
+    etag_file.sync_all()?;
 
     // trigger recompilation when a new migration is added
     println!("cargo:rerun-if-changed=migrations");
@@ -118,6 +132,16 @@ fn get_git_hash() -> Result<Option<String>> {
     }
 }
 
+fn etag_from_path(path: impl AsRef<Path>) -> Result<String> {
+    Ok(etag_from_content(std::fs::read(&path)?))
+}
+
+fn etag_from_content(content: impl AsRef<[u8]>) -> String {
+    let digest = md5::compute(content);
+    let md5_hex = format!("{:x}", digest);
+    format!(r#""\"{md5_hex}\"""#)
+}
+
 fn compile_sass_file(src: &Path, dest: &Path) -> Result<()> {
     let css = grass::from_path(
         src.to_str()
@@ -133,7 +157,7 @@ fn compile_sass_file(src: &Path, dest: &Path) -> Result<()> {
     Ok(())
 }
 
-fn compile_sass(out_dir: &Path) -> Result<()> {
+fn compile_sass(out_dir: &Path, etag_map: &mut ETagMap) -> Result<()> {
     const STYLE_DIR: &str = "templates/style";
 
     for entry in walkdir::WalkDir::new(STYLE_DIR) {
@@ -146,12 +170,13 @@ fn compile_sass(out_dir: &Path) -> Result<()> {
                 .to_str()
                 .context("file name must be a utf-8 string")?;
             if !file_name.starts_with('_') {
-                let dest = out_dir
-                    .join(entry.path().strip_prefix(STYLE_DIR)?)
-                    .with_extension("css");
+                let dest = out_dir.join(file_name).with_extension("css");
                 compile_sass_file(entry.path(), &dest).with_context(|| {
                     format!("compiling {} to {}", entry.path().display(), dest.display())
                 })?;
+
+                let dest_str = dest.file_name().unwrap().to_str().unwrap().to_owned();
+                etag_map.entry(dest_str, etag_from_path(&dest)?);
             }
         }
     }
@@ -160,7 +185,32 @@ fn compile_sass(out_dir: &Path) -> Result<()> {
     let pure = tracked::read_to_string("vendor/pure-css/css/pure-min.css")?;
     let grids = tracked::read_to_string("vendor/pure-css/css/grids-responsive-min.css")?;
     let vendored = pure + &grids;
-    std::fs::write(out_dir.join("vendored").with_extension("css"), vendored)?;
+    std::fs::write(out_dir.join("vendored").with_extension("css"), &vendored)?;
+
+    etag_map.entry(
+        "vendored.css".to_owned(),
+        etag_from_content(vendored.as_bytes()),
+    );
+
+    Ok(())
+}
+
+fn calculate_static_etags(etag_map: &mut ETagMap) -> Result<()> {
+    const STATIC_DIRS: &[&str] = &["static", "vendor"];
+
+    for static_dir in STATIC_DIRS {
+        for entry in walkdir::WalkDir::new(static_dir) {
+            let entry = entry?;
+            let path = entry.path();
+            if !path.is_file() {
+                continue;
+            }
+
+            let partial_path = path.strip_prefix(static_dir).unwrap();
+            let partial_path_str = partial_path.to_string_lossy().to_string();
+            etag_map.entry(partial_path_str, etag_from_path(path)?);
+        }
+    }
 
     Ok(())
 }

build.rs

@@ -10,6 +10,10 @@ macro_rules! mime {
 mime!(APPLICATION_ZIP, "application/zip");
 mime!(APPLICATION_ZSTD, "application/zstd");
 mime!(APPLICATION_GZIP, "application/gzip");
+mime!(
+    APPLICATION_OPENSEARCH_XML,
+    "application/opensearchdescription+xml"
+);
 mime!(APPLICATION_XML, "application/xml");
 mime!(TEXT_MARKDOWN, "text/markdown");
 mime!(TEXT_RUST, "text/rust");

src/db/mimes.rs

@@ -0,0 +1,173 @@
+//! Adapted version of `headers::IfNoneMatch`.
+//!
+//! The combination of `TypedHeader` and `IfNoneMatch` works in odd ways.
+//! They are built in a way that a _missing_ `If-None-Match` header will lead to:
+//!
+//! 1. extractor with `TypedHeader<IfNoneMatch>` returning `IfNoneMatch("")`
+//! 2. extractor with `Option<TypedHeader<IfNoneMatch>>` returning `Some(IfNoneMatch(""))`
+//!
+//! Where I would expect:
+//! 1. a failure because of the missing header
+//! 2. `None` for the missing header
+//!
+//! This could be solved by either adapting `TypedHeader` or `IfNoneMatch`, I'm not sure which is
+//! right.
+//!
+//! Some reading material for those interested:
+//! * https://github.com/hyperium/headers/issues/204
+//! * https://github.com/hyperium/headers/pull/165
+//! * https://github.com/tokio-rs/axum/issues/1781
+//! * https://github.com/tokio-rs/axum/pull/1810
+//! * https://github.com/tokio-rs/axum/pull/2475
+//!
+//! Right now I feel like adapting `IfNoneMatch` is the "most correct-ish" option.
+
+#[allow(clippy::disallowed_types)]
+mod header_impl {
+    use axum_extra::headers::{self, ETag, Header, IfNoneMatch as OriginalIfNoneMatch};
+    use derive_more::Deref;
+
+    #[derive(Debug, Clone, PartialEq, Deref)]
+    pub(crate) struct IfNoneMatch(pub axum_extra::headers::IfNoneMatch);
+
+    impl Header for IfNoneMatch {
+        fn name() -> &'static http::HeaderName {
+            OriginalIfNoneMatch::name()
+        }
+
+        fn decode<'i, I>(values: &mut I) -> Result<Self, headers::Error>
+        where
+            Self: Sized,
+            I: Iterator<Item = &'i http::HeaderValue>,
+        {
+            let mut values = values.peekable();
+
+            // NOTE: this is the difference to the original implementation.
+            // When there is no header in the request, I want the decoding to fail.
+            // This makes Option<TypedHeader<H>> return `None`, and also matches
+            // most other header implementations.
+            if values.peek().is_none() {
+                Err(headers::Error::invalid())
+            } else {
+                OriginalIfNoneMatch::decode(&mut values).map(IfNoneMatch)
+            }
+        }
+
+        fn encode<E: Extend<http::HeaderValue>>(&self, values: &mut E) {
+            self.0.encode(values)
+        }
+    }
+
+    impl From<ETag> for IfNoneMatch {
+        fn from(value: ETag) -> Self {
+            Self(value.into())
+        }
+    }
+}
+
+pub(crate) use header_impl::IfNoneMatch;
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use anyhow::Result;
+    use axum::{RequestPartsExt, body::Body, extract::Request};
+    use axum_extra::{
+        TypedHeader,
+        headers::{ETag, HeaderMapExt as _},
+    };
+    use http::{HeaderMap, request};
+
+    fn parts(if_none_match: Option<IfNoneMatch>) -> request::Parts {
+        let mut builder = Request::builder();
+
+        if let Some(if_none_match) = if_none_match {
+            let headers = builder.headers_mut().unwrap();
+            headers.typed_insert(if_none_match.clone());
+        }
+
+        let (parts, _body) = builder.uri("/").body(Body::empty()).unwrap().into_parts();
+
+        parts
+    }
+
+    fn example_header() -> IfNoneMatch {
+        IfNoneMatch::from("\"some-etag-value\"".parse::<ETag>().unwrap())
+    }
+
+    #[test]
+    fn test_normal_typed_get_with_empty_headers() {
+        let map = HeaderMap::new();
+        assert!(map.typed_get::<IfNoneMatch>().is_none());
+        assert!(map.typed_try_get::<IfNoneMatch>().unwrap().is_none());
+    }
+
+    #[test]
+    fn test_normal_typed_get_with_value_headers() -> Result<()> {
+        let if_none_match = example_header();
+
+        let mut map = HeaderMap::new();
+        map.typed_insert(if_none_match.clone());
+
+        assert_eq!(map.typed_get::<IfNoneMatch>(), Some(if_none_match.clone()));
+        assert_eq!(map.typed_try_get::<IfNoneMatch>()?, Some(if_none_match));
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_extract_from_empty_request_via_optional_typed_header() -> Result<()> {
+        let mut parts = parts(None);
+
+        assert!(
+            parts
+                .extract::<Option<TypedHeader<IfNoneMatch>>>()
+                .await?
+                // this is what we want, and the default `headers::IfNoneMatch` header can't
+                // offer. Or the impl of  the `TypedHeader` extractor, depending on
+                // interpretation.
+                .is_none()
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_extract_from_empty_request_via_mandatory_typed_header() -> Result<()> {
+        let mut parts = parts(None);
+
+        // mandatory extractor leads to error when the header is missing.
+        assert!(parts.extract::<TypedHeader<IfNoneMatch>>().await.is_err());
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_extract_from_header_via_optional_typed_header() -> Result<()> {
+        let if_none_match = example_header();
+        let mut parts = parts(Some(if_none_match.clone()));
+
+        assert_eq!(
+            parts
+                .extract::<Option<TypedHeader<IfNoneMatch>>>()
+                .await?
+                .map(|th| th.0),
+            Some(if_none_match)
+        );
+
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn test_extract_from_header_via_mandatory_typed_header() -> Result<()> {
+        let if_none_match = example_header();
+        let mut parts = parts(Some(if_none_match.clone()));
+
+        assert_eq!(
+            parts.extract::<TypedHeader<IfNoneMatch>>().await?.0,
+            if_none_match
+        );
+
+        Ok(())
+    }
+}