From 8c68d5c15751baa01a05436759ddf83f1e09abf9 Mon Sep 17 00:00:00 2001 From: Peter Holloway Date: Thu, 13 Feb 2025 09:08:21 +0000 Subject: [PATCH 1/7] Add initial methods for storing and retrieving templates --- ...f032ed741abbd5f26fb9dd0c3079a313ef958.json | 12 ++++++ ...3d19c75504c917b65e59ae7f396508251079b.json | 26 ++++++++++++ migrations/0003_template_table.down.sql | 1 + migrations/0003_template_table.up.sql | 7 ++++ src/db_service.rs | 41 ++++++++++++++++++- 5 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 .sqlx/query-74ddd4d18b2b3529839b609905af032ed741abbd5f26fb9dd0c3079a313ef958.json create mode 100644 .sqlx/query-f3b05af5405d08d1577c12fbc483d19c75504c917b65e59ae7f396508251079b.json create mode 100644 migrations/0003_template_table.down.sql create mode 100644 migrations/0003_template_table.up.sql diff --git a/.sqlx/query-74ddd4d18b2b3529839b609905af032ed741abbd5f26fb9dd0c3079a313ef958.json b/.sqlx/query-74ddd4d18b2b3529839b609905af032ed741abbd5f26fb9dd0c3079a313ef958.json new file mode 100644 index 00000000..54721760 --- /dev/null +++ b/.sqlx/query-74ddd4d18b2b3529839b609905af032ed741abbd5f26fb9dd0c3079a313ef958.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO templates (name, template, beamline) VALUES (?, ?, (SELECT id FROM beamline WHERE name = ?));", + "describe": { + "columns": [], + "parameters": { + "Right": 3 + }, + "nullable": [] + }, + "hash": "74ddd4d18b2b3529839b609905af032ed741abbd5f26fb9dd0c3079a313ef958" +} diff --git a/.sqlx/query-f3b05af5405d08d1577c12fbc483d19c75504c917b65e59ae7f396508251079b.json b/.sqlx/query-f3b05af5405d08d1577c12fbc483d19c75504c917b65e59ae7f396508251079b.json new file mode 100644 index 00000000..97b090ba --- /dev/null +++ b/.sqlx/query-f3b05af5405d08d1577c12fbc483d19c75504c917b65e59ae7f396508251079b.json @@ -0,0 +1,26 @@ +{ + "db_name": "SQLite", + "query": "SELECT templates.name, templates.template\n FROM beamline JOIN templates ON beamline.id = templates.beamline\n WHERE beamline.name = ?", + "describe": { + "columns": [ + { + "name": "name", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "template", + "ordinal": 1, + "type_info": "Text" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false + ] + }, + "hash": "f3b05af5405d08d1577c12fbc483d19c75504c917b65e59ae7f396508251079b" +} diff --git a/migrations/0003_template_table.down.sql b/migrations/0003_template_table.down.sql new file mode 100644 index 00000000..39e152d2 --- /dev/null +++ b/migrations/0003_template_table.down.sql @@ -0,0 +1 @@ +DROP TABLE templates; diff --git a/migrations/0003_template_table.up.sql b/migrations/0003_template_table.up.sql new file mode 100644 index 00000000..0a39e6ea --- /dev/null +++ b/migrations/0003_template_table.up.sql @@ -0,0 +1,7 @@ +-- Add new table for additional templates +CREATE TABLE templates ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL CHECK (length(name) > 0), + template TEXT NOT NULL CHECK (length(name) > 0), + beamline INTEGER NOT NULL REFERENCES beamline(id) ON DELETE CASCADE ON UPDATE CASCADE +) diff --git a/src/db_service.rs b/src/db_service.rs index 04450149..5bbe1445 100644 --- a/src/db_service.rs +++ b/src/db_service.rs @@ -19,7 +19,7 @@ use std::path::Path; pub use error::ConfigurationError; use error::NewConfigurationError; use sqlx::sqlite::{SqliteConnectOptions, SqliteRow}; -use sqlx::{query_as, FromRow, QueryBuilder, Row, Sqlite, SqlitePool}; +use sqlx::{query, query_as, FromRow, QueryBuilder, Row, Sqlite, SqlitePool}; use tracing::{info, instrument, trace}; use crate::paths::{ @@ -35,6 +35,12 @@ pub struct SqliteScanPathService { pool: SqlitePool, } +#[derive(Debug)] +pub struct NamedTemplate { + pub name: String, + pub template: String, +} + #[derive(Debug, PartialEq, Eq)] struct RawPathTemplate(String, PhantomData); @@ -341,6 +347,39 @@ impl SqliteScanPathService { .ok_or(ConfigurationError::MissingInstrument(instrument.into())) } + pub async fn additional_templates( + &self, + beamline: &str, + ) -> Result, ConfigurationError> { + Ok(query_as!( + NamedTemplate, + "SELECT templates.name, templates.template + FROM beamline JOIN templates ON beamline.id = templates.beamline + WHERE beamline.name = ?", + beamline + ) + .fetch_all(&self.pool) + .await?) + } + + pub async fn register_template( + &self, + beamline: &str, + name: String, + template: String, + ) -> Result<(), ConfigurationError> { + query!( + "INSERT INTO templates (name, template, beamline) + VALUES (?, ?, (SELECT id FROM beamline WHERE name = ?));", + name, + template, + beamline + ) + .execute(&self.pool) + .await?; + Ok(()) + } + /// Create a db service from a new empty/schema-less DB #[cfg(test)] pub(crate) async fn uninitialised() -> Self { From 97e0295bc690d8330f487e8bb548513cb1b251a3 Mon Sep 17 00:00:00 2001 From: Peter Holloway Date: Thu, 13 Feb 2025 16:04:59 +0000 Subject: [PATCH 2/7] Working named template registration/retrieval Still can't do anything with the templates though. And they're not validated --- ...3379f45ec83a36351fc19353f8f48262c79e.json} | 4 +- ...f032ed741abbd5f26fb9dd0c3079a313ef958.json | 12 --- ...99bf4d9e2893f77b02765f9c9712f74b0e56b.json | 26 ++++++ migrations/0003_template_table.down.sql | 3 +- migrations/0003_template_table.up.sql | 22 ++++- src/db_service.rs | 92 ++++++++++++++++--- src/graphql/mod.rs | 51 +++++++++- 7 files changed, 176 insertions(+), 34 deletions(-) rename .sqlx/{query-f3b05af5405d08d1577c12fbc483d19c75504c917b65e59ae7f396508251079b.json => query-15bdb26777072892c7feb712ae373379f45ec83a36351fc19353f8f48262c79e.json} (57%) delete mode 100644 .sqlx/query-74ddd4d18b2b3529839b609905af032ed741abbd5f26fb9dd0c3079a313ef958.json create mode 100644 .sqlx/query-b09ef7bc16a56728acaf3b4a9e899bf4d9e2893f77b02765f9c9712f74b0e56b.json diff --git a/.sqlx/query-f3b05af5405d08d1577c12fbc483d19c75504c917b65e59ae7f396508251079b.json b/.sqlx/query-15bdb26777072892c7feb712ae373379f45ec83a36351fc19353f8f48262c79e.json similarity index 57% rename from .sqlx/query-f3b05af5405d08d1577c12fbc483d19c75504c917b65e59ae7f396508251079b.json rename to .sqlx/query-15bdb26777072892c7feb712ae373379f45ec83a36351fc19353f8f48262c79e.json index 97b090ba..77439d0b 100644 --- a/.sqlx/query-f3b05af5405d08d1577c12fbc483d19c75504c917b65e59ae7f396508251079b.json +++ b/.sqlx/query-15bdb26777072892c7feb712ae373379f45ec83a36351fc19353f8f48262c79e.json @@ -1,6 +1,6 @@ { "db_name": "SQLite", - "query": "SELECT templates.name, templates.template\n FROM beamline JOIN templates ON beamline.id = templates.beamline\n WHERE beamline.name = ?", + "query": "SELECT name, template FROM beamline_template WHERE beamline = ?", "describe": { "columns": [ { @@ -22,5 +22,5 @@ false ] }, - "hash": "f3b05af5405d08d1577c12fbc483d19c75504c917b65e59ae7f396508251079b" + "hash": "15bdb26777072892c7feb712ae373379f45ec83a36351fc19353f8f48262c79e" } diff --git a/.sqlx/query-74ddd4d18b2b3529839b609905af032ed741abbd5f26fb9dd0c3079a313ef958.json b/.sqlx/query-74ddd4d18b2b3529839b609905af032ed741abbd5f26fb9dd0c3079a313ef958.json deleted file mode 100644 index 54721760..00000000 --- a/.sqlx/query-74ddd4d18b2b3529839b609905af032ed741abbd5f26fb9dd0c3079a313ef958.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "INSERT INTO templates (name, template, beamline) VALUES (?, ?, (SELECT id FROM beamline WHERE name = ?));", - "describe": { - "columns": [], - "parameters": { - "Right": 3 - }, - "nullable": [] - }, - "hash": "74ddd4d18b2b3529839b609905af032ed741abbd5f26fb9dd0c3079a313ef958" -} diff --git a/.sqlx/query-b09ef7bc16a56728acaf3b4a9e899bf4d9e2893f77b02765f9c9712f74b0e56b.json b/.sqlx/query-b09ef7bc16a56728acaf3b4a9e899bf4d9e2893f77b02765f9c9712f74b0e56b.json new file mode 100644 index 00000000..8bc6250c --- /dev/null +++ b/.sqlx/query-b09ef7bc16a56728acaf3b4a9e899bf4d9e2893f77b02765f9c9712f74b0e56b.json @@ -0,0 +1,26 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO template (name, template, beamline)\n VALUES (?, ?, (SELECT id FROM beamline WHERE name = ?)) RETURNING name, template;", + "describe": { + "columns": [ + { + "name": "name", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "template", + "ordinal": 1, + "type_info": "Text" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false + ] + }, + "hash": "b09ef7bc16a56728acaf3b4a9e899bf4d9e2893f77b02765f9c9712f74b0e56b" +} diff --git a/migrations/0003_template_table.down.sql b/migrations/0003_template_table.down.sql index 39e152d2..59b9850d 100644 --- a/migrations/0003_template_table.down.sql +++ b/migrations/0003_template_table.down.sql @@ -1 +1,2 @@ -DROP TABLE templates; +DROP VIEW beamline_template; +DROP TABLE template; diff --git a/migrations/0003_template_table.up.sql b/migrations/0003_template_table.up.sql index 0a39e6ea..138585c9 100644 --- a/migrations/0003_template_table.up.sql +++ b/migrations/0003_template_table.up.sql @@ -1,7 +1,19 @@ -- Add new table for additional templates -CREATE TABLE templates ( +CREATE TABLE template ( id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL CHECK (length(name) > 0), - template TEXT NOT NULL CHECK (length(name) > 0), - beamline INTEGER NOT NULL REFERENCES beamline(id) ON DELETE CASCADE ON UPDATE CASCADE -) + name TEXT NOT NULL, + template TEXT NOT NULL, + beamline INTEGER NOT NULL REFERENCES beamline(id) ON DELETE CASCADE ON UPDATE CASCADE, + + CONSTRAINT duplicate_names UNIQUE (name, beamline) ON CONFLICT REPLACE, + + CONSTRAINT empty_template CHECK (length(template) > 0), + CONSTRAINT empty_name CHECK (length(name) > 0) +); + +CREATE VIEW beamline_template (beamline, name, template) AS + SELECT + beamline.name, template.name, template.template + FROM beamline + JOIN template + ON beamline.id = template.beamline; diff --git a/src/db_service.rs b/src/db_service.rs index 5bbe1445..8e54f729 100644 --- a/src/db_service.rs +++ b/src/db_service.rs @@ -16,10 +16,10 @@ use std::fmt; use std::marker::PhantomData; use std::path::Path; -pub use error::ConfigurationError; use error::NewConfigurationError; +pub use error::{ConfigurationError, NamedTemplateError}; use sqlx::sqlite::{SqliteConnectOptions, SqliteRow}; -use sqlx::{query, query_as, FromRow, QueryBuilder, Row, Sqlite, SqlitePool}; +use sqlx::{query_as, FromRow, QueryBuilder, Row, Sqlite, SqlitePool}; use tracing::{info, instrument, trace}; use crate::paths::{ @@ -41,6 +41,15 @@ pub struct NamedTemplate { pub template: String, } +impl<'r> FromRow<'r, SqliteRow> for NamedTemplate { + fn from_row(row: &'r SqliteRow) -> Result { + Ok(Self { + name: row.try_get("name")?, + template: row.try_get("template")?, + }) + } +} + #[derive(Debug, PartialEq, Eq)] struct RawPathTemplate(String, PhantomData); @@ -347,37 +356,52 @@ impl SqliteScanPathService { .ok_or(ConfigurationError::MissingInstrument(instrument.into())) } - pub async fn additional_templates( + pub async fn all_additional_templates( &self, beamline: &str, ) -> Result, ConfigurationError> { Ok(query_as!( NamedTemplate, - "SELECT templates.name, templates.template - FROM beamline JOIN templates ON beamline.id = templates.beamline - WHERE beamline.name = ?", + "SELECT name, template FROM beamline_template WHERE beamline = ?", beamline ) .fetch_all(&self.pool) .await?) } + pub async fn additional_templates( + &self, + beamline: &str, + names: Vec, + ) -> Result, ConfigurationError> { + let mut q = + QueryBuilder::new("SELECT name, template FROM beamline_template WHERE beamline = "); + q.push_bind(beamline); + q.push(" AND name IN ("); + let mut name_query = q.separated(", "); + for name in names { + name_query.push_bind(name); + } + q.push(")"); + let query = q.build_query_as(); + Ok(query.fetch_all(&self.pool).await?) + } pub async fn register_template( &self, beamline: &str, name: String, template: String, - ) -> Result<(), ConfigurationError> { - query!( - "INSERT INTO templates (name, template, beamline) - VALUES (?, ?, (SELECT id FROM beamline WHERE name = ?));", + ) -> Result { + Ok(query_as!( + NamedTemplate, + "INSERT INTO template (name, template, beamline) + VALUES (?, ?, (SELECT id FROM beamline WHERE name = ?)) RETURNING name, template;", name, template, beamline ) - .execute(&self.pool) - .await?; - Ok(()) + .fetch_one(&self.pool) + .await?) } /// Create a db service from a new empty/schema-less DB @@ -407,7 +431,9 @@ impl fmt::Debug for SqliteScanPathService { } mod error { + use derive_more::{Display, Error, From}; + use sqlx::error::ErrorKind; #[derive(Debug, Display, Error, From)] pub enum ConfigurationError { @@ -431,6 +457,46 @@ mod error { Self::MissingField(value.into()) } } + + #[derive(Debug, Display, Error)] + pub enum NamedTemplateError { + #[display("No configuration for beamline")] + MissingBeamline, + #[display("Template name was empty")] + EmptyName, + #[display("Template was empty")] + EmptyTemplate, + #[display("Error accessing named template: {_0}")] + DbError(sqlx::Error), + } + + impl From for NamedTemplateError { + fn from(value: sqlx::Error) -> Self { + match value { + sqlx::Error::Database(err) => match (err.kind(), err.message().split_once(": ")) { + (ErrorKind::NotNullViolation, Some((_, "template.beamline"))) => { + NamedTemplateError::MissingBeamline + } + // pretty sure these two are not possible as strings can't be null + (ErrorKind::NotNullViolation, Some((_, "template.name"))) => { + NamedTemplateError::EmptyName + } + (ErrorKind::NotNullViolation, Some((_, "template.template"))) => { + NamedTemplateError::EmptyTemplate + } + // Values are empty - these rely on the named checks in the schema + (ErrorKind::CheckViolation, Some((_, "empty_name"))) => { + NamedTemplateError::EmptyName + } + (ErrorKind::CheckViolation, Some((_, "empty_template"))) => { + NamedTemplateError::EmptyTemplate + } + (_, _) => NamedTemplateError::DbError(sqlx::Error::Database(err)), + }, + err => err.into(), + } + } + } } #[cfg(test)] diff --git a/src/graphql/mod.rs b/src/graphql/mod.rs index 6edc8ca2..0ef24f65 100644 --- a/src/graphql/mod.rs +++ b/src/graphql/mod.rs @@ -41,7 +41,7 @@ use tracing::{debug, info, instrument, trace, warn}; use crate::build_info::ServerStatus; use crate::cli::ServeOptions; use crate::db_service::{ - InstrumentConfiguration, InstrumentConfigurationUpdate, SqliteScanPathService, + InstrumentConfiguration, InstrumentConfigurationUpdate, NamedTemplate, SqliteScanPathService, }; use crate::numtracker::NumTracker; use crate::paths::{ @@ -302,6 +302,23 @@ impl CurrentConfiguration { } } +#[derive(Debug, InputObject)] +struct TemplateInput { + name: String, + template: String, +} + +#[Object] +impl NamedTemplate { + async fn name(&self) -> &str { + &self.name + } + + async fn template(&self) -> &str { + &self.template + } +} + impl FieldSource for ScanPaths { fn resolve(&self, field: &ScanField) -> Cow<'_, str> { match field { @@ -384,6 +401,24 @@ impl Query { .into_iter() .collect() } + + #[instrument(skip(self, ctx))] + async fn named_templates<'ctx>( + &self, + ctx: &Context<'ctx>, + beamline: String, + names: Option>, + ) -> async_graphql::Result> { + check_auth(ctx, |policy, token| { + policy.check_beamline_admin(token, &beamline) + }) + .await?; + let db = ctx.data::()?; + match names { + Some(names) => Ok(db.additional_templates(&beamline, names).await?), + None => Ok(db.all_additional_templates(&beamline).await?), + } + } } #[Object] @@ -451,6 +486,20 @@ impl Mutation { }; CurrentConfiguration::for_config(db_config, nt).await } + + #[instrument(skip(self, ctx))] + async fn register_template<'ctx>( + &self, + ctx: &Context<'ctx>, + beamline: String, + template: TemplateInput, + ) -> async_graphql::Result { + check_auth(ctx, |pc, token| pc.check_beamline_admin(token, &beamline)).await?; + let db = ctx.data::()?; + Ok(db + .register_template(&beamline, template.name, template.template) + .await?) + } } async fn check_auth<'ctx, Check, R>(ctx: &Context<'ctx>, check: Check) -> async_graphql::Result<()> From 042e259a57745e555fee24520d28f8b4ecdf813e Mon Sep 17 00:00:00 2001 From: Peter Holloway Date: Thu, 13 Feb 2025 16:53:01 +0000 Subject: [PATCH 3/7] Working named templating Assuming everything is used correctly. No error handling yet. --- src/graphql/mod.rs | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/src/graphql/mod.rs b/src/graphql/mod.rs index 0ef24f65..a1ae4ae0 100644 --- a/src/graphql/mod.rs +++ b/src/graphql/mod.rs @@ -14,6 +14,7 @@ use std::any; use std::borrow::Cow; +use std::collections::HashMap; use std::future::Future; use std::io::Write; use std::path::{Component, PathBuf}; @@ -144,6 +145,7 @@ struct DirectoryPath { /// GraphQL type to provide path data for the next scan for a given instrument session struct ScanPaths { directory: DirectoryPath, + extra_templates: HashMap>, subdirectory: Subdirectory, } @@ -224,6 +226,12 @@ impl ScanPaths { self.directory.info.scan_number() } + async fn template(&self, name: String) -> async_graphql::Result { + Ok(path_to_string( + self.extra_templates.get(&name).unwrap().render(self), + )?) + } + /// The paths where the given detectors should write their files. /// /// Detector names are normalised before being used in file names by replacing any @@ -410,7 +418,7 @@ impl Query { names: Option>, ) -> async_graphql::Result> { check_auth(ctx, |policy, token| { - policy.check_beamline_admin(token, &beamline) + policy.check_instrument_admin(token, &beamline) }) .await?; let db = ctx.data::()?; @@ -455,11 +463,38 @@ impl Mutation { warn!("Failed to increment tracker file: {e}"); } + let required_templates = ctx + .field() + .selection_set() + .filter(|slct| slct.name() == "template") + .flat_map(|slct| slct.arguments()) + .filter_map(|args| { + args.get(0).map(|arg| { + let Value::String(name) = &arg.1 else { + panic!("name isn't a string") + }; + name.into() + }) + }) + .collect::>(); + let extra_templates = db + .additional_templates(&instrument, required_templates) + .await? + .into_iter() + .map(|template| { + ( + template.name, + ScanTemplate::new_checked(&template.template).unwrap(), + ) + }) + .collect(); + Ok(ScanPaths { directory: DirectoryPath { instrument_session, info: next_scan, }, + extra_templates, subdirectory: sub.unwrap_or_default(), }) } @@ -494,7 +529,7 @@ impl Mutation { beamline: String, template: TemplateInput, ) -> async_graphql::Result { - check_auth(ctx, |pc, token| pc.check_beamline_admin(token, &beamline)).await?; + check_auth(ctx, |pc, token| pc.check_instrument_admin(token, &beamline)).await?; let db = ctx.data::()?; Ok(db .register_template(&beamline, template.name, template.template) From 11529dd6923f20eeaa3257f1f702e4f6e56bb998 Mon Sep 17 00:00:00 2001 From: Peter Holloway Date: Tue, 26 Aug 2025 17:38:17 +0100 Subject: [PATCH 4/7] Update post rebase --- ...b9423c1b63146c2d93e3f1b08857f28a6c33.json} | 4 +- ...7f86fe119aa5b5d7f36e49bece9ab6048c38.json} | 4 +- migrations/0003_template_table.down.sql | 2 - migrations/0003_template_table.up.sql | 19 -- migrations/0005_template_table.down.sql | 2 + migrations/0005_template_table.up.sql | 19 ++ src/db_service.rs | 28 +-- static/service_schema.graphql | 195 ------------------ 8 files changed, 39 insertions(+), 234 deletions(-) rename .sqlx/{query-b09ef7bc16a56728acaf3b4a9e899bf4d9e2893f77b02765f9c9712f74b0e56b.json => query-03cac7cb5b485c93b71cdbe03400b9423c1b63146c2d93e3f1b08857f28a6c33.json} (59%) rename .sqlx/{query-15bdb26777072892c7feb712ae373379f45ec83a36351fc19353f8f48262c79e.json => query-4323ce25def2ee41c1019820cad67f86fe119aa5b5d7f36e49bece9ab6048c38.json} (68%) delete mode 100644 migrations/0003_template_table.down.sql delete mode 100644 migrations/0003_template_table.up.sql create mode 100644 migrations/0005_template_table.down.sql create mode 100644 migrations/0005_template_table.up.sql diff --git a/.sqlx/query-b09ef7bc16a56728acaf3b4a9e899bf4d9e2893f77b02765f9c9712f74b0e56b.json b/.sqlx/query-03cac7cb5b485c93b71cdbe03400b9423c1b63146c2d93e3f1b08857f28a6c33.json similarity index 59% rename from .sqlx/query-b09ef7bc16a56728acaf3b4a9e899bf4d9e2893f77b02765f9c9712f74b0e56b.json rename to .sqlx/query-03cac7cb5b485c93b71cdbe03400b9423c1b63146c2d93e3f1b08857f28a6c33.json index 8bc6250c..a561a5c2 100644 --- a/.sqlx/query-b09ef7bc16a56728acaf3b4a9e899bf4d9e2893f77b02765f9c9712f74b0e56b.json +++ b/.sqlx/query-03cac7cb5b485c93b71cdbe03400b9423c1b63146c2d93e3f1b08857f28a6c33.json @@ -1,6 +1,6 @@ { "db_name": "SQLite", - "query": "INSERT INTO template (name, template, beamline)\n VALUES (?, ?, (SELECT id FROM beamline WHERE name = ?)) RETURNING name, template;", + "query": "INSERT INTO template (name, template, instrument)\n VALUES (?, ?, (SELECT id FROM instrument WHERE name = ?)) RETURNING name, template;", "describe": { "columns": [ { @@ -22,5 +22,5 @@ false ] }, - "hash": "b09ef7bc16a56728acaf3b4a9e899bf4d9e2893f77b02765f9c9712f74b0e56b" + "hash": "03cac7cb5b485c93b71cdbe03400b9423c1b63146c2d93e3f1b08857f28a6c33" } diff --git a/.sqlx/query-15bdb26777072892c7feb712ae373379f45ec83a36351fc19353f8f48262c79e.json b/.sqlx/query-4323ce25def2ee41c1019820cad67f86fe119aa5b5d7f36e49bece9ab6048c38.json similarity index 68% rename from .sqlx/query-15bdb26777072892c7feb712ae373379f45ec83a36351fc19353f8f48262c79e.json rename to .sqlx/query-4323ce25def2ee41c1019820cad67f86fe119aa5b5d7f36e49bece9ab6048c38.json index 77439d0b..39be7927 100644 --- a/.sqlx/query-15bdb26777072892c7feb712ae373379f45ec83a36351fc19353f8f48262c79e.json +++ b/.sqlx/query-4323ce25def2ee41c1019820cad67f86fe119aa5b5d7f36e49bece9ab6048c38.json @@ -1,6 +1,6 @@ { "db_name": "SQLite", - "query": "SELECT name, template FROM beamline_template WHERE beamline = ?", + "query": "SELECT name, template FROM instrument_template WHERE instrument = ?", "describe": { "columns": [ { @@ -22,5 +22,5 @@ false ] }, - "hash": "15bdb26777072892c7feb712ae373379f45ec83a36351fc19353f8f48262c79e" + "hash": "4323ce25def2ee41c1019820cad67f86fe119aa5b5d7f36e49bece9ab6048c38" } diff --git a/migrations/0003_template_table.down.sql b/migrations/0003_template_table.down.sql deleted file mode 100644 index 59b9850d..00000000 --- a/migrations/0003_template_table.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP VIEW beamline_template; -DROP TABLE template; diff --git a/migrations/0003_template_table.up.sql b/migrations/0003_template_table.up.sql deleted file mode 100644 index 138585c9..00000000 --- a/migrations/0003_template_table.up.sql +++ /dev/null @@ -1,19 +0,0 @@ --- Add new table for additional templates -CREATE TABLE template ( - id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL, - template TEXT NOT NULL, - beamline INTEGER NOT NULL REFERENCES beamline(id) ON DELETE CASCADE ON UPDATE CASCADE, - - CONSTRAINT duplicate_names UNIQUE (name, beamline) ON CONFLICT REPLACE, - - CONSTRAINT empty_template CHECK (length(template) > 0), - CONSTRAINT empty_name CHECK (length(name) > 0) -); - -CREATE VIEW beamline_template (beamline, name, template) AS - SELECT - beamline.name, template.name, template.template - FROM beamline - JOIN template - ON beamline.id = template.beamline; diff --git a/migrations/0005_template_table.down.sql b/migrations/0005_template_table.down.sql new file mode 100644 index 00000000..9f22b14d --- /dev/null +++ b/migrations/0005_template_table.down.sql @@ -0,0 +1,2 @@ +DROP VIEW instrument_template; +DROP TABLE template; diff --git a/migrations/0005_template_table.up.sql b/migrations/0005_template_table.up.sql new file mode 100644 index 00000000..f0dab661 --- /dev/null +++ b/migrations/0005_template_table.up.sql @@ -0,0 +1,19 @@ +-- Add new table for additional templates +CREATE TABLE template ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + template TEXT NOT NULL, + instrument INTEGER NOT NULL REFERENCES instrument(id) ON DELETE CASCADE ON UPDATE CASCADE, + + CONSTRAINT duplicate_names UNIQUE (name, instrument) ON CONFLICT REPLACE, + + CONSTRAINT empty_template CHECK (length(template) > 0), + CONSTRAINT empty_name CHECK (length(name) > 0) +); + +CREATE VIEW instrument_template (instrument, name, template) AS + SELECT + instrument.name, template.name, template.template + FROM instrument + JOIN template + ON instrument.id = template.instrument; diff --git a/src/db_service.rs b/src/db_service.rs index 8e54f729..4e01ef49 100644 --- a/src/db_service.rs +++ b/src/db_service.rs @@ -358,24 +358,24 @@ impl SqliteScanPathService { pub async fn all_additional_templates( &self, - beamline: &str, + instrument: &str, ) -> Result, ConfigurationError> { Ok(query_as!( NamedTemplate, - "SELECT name, template FROM beamline_template WHERE beamline = ?", - beamline + "SELECT name, template FROM instrument_template WHERE instrument = ?", + instrument ) .fetch_all(&self.pool) .await?) } pub async fn additional_templates( &self, - beamline: &str, + instrument: &str, names: Vec, ) -> Result, ConfigurationError> { let mut q = - QueryBuilder::new("SELECT name, template FROM beamline_template WHERE beamline = "); - q.push_bind(beamline); + QueryBuilder::new("SELECT name, template FROM instrument_template WHERE instrument = "); + q.push_bind(instrument); q.push(" AND name IN ("); let mut name_query = q.separated(", "); for name in names { @@ -388,17 +388,17 @@ impl SqliteScanPathService { pub async fn register_template( &self, - beamline: &str, + instrument: &str, name: String, template: String, ) -> Result { Ok(query_as!( NamedTemplate, - "INSERT INTO template (name, template, beamline) - VALUES (?, ?, (SELECT id FROM beamline WHERE name = ?)) RETURNING name, template;", + "INSERT INTO template (name, template, instrument) + VALUES (?, ?, (SELECT id FROM instrument WHERE name = ?)) RETURNING name, template;", name, template, - beamline + instrument ) .fetch_one(&self.pool) .await?) @@ -460,8 +460,8 @@ mod error { #[derive(Debug, Display, Error)] pub enum NamedTemplateError { - #[display("No configuration for beamline")] - MissingBeamline, + #[display("No configuration for instrument")] + MissingInstrument, #[display("Template name was empty")] EmptyName, #[display("Template was empty")] @@ -474,8 +474,8 @@ mod error { fn from(value: sqlx::Error) -> Self { match value { sqlx::Error::Database(err) => match (err.kind(), err.message().split_once(": ")) { - (ErrorKind::NotNullViolation, Some((_, "template.beamline"))) => { - NamedTemplateError::MissingBeamline + (ErrorKind::NotNullViolation, Some((_, "template.instrument"))) => { + NamedTemplateError::MissingInstrument } // pretty sure these two are not possible as strings can't be null (ErrorKind::NotNullViolation, Some((_, "template.name"))) => { diff --git a/static/service_schema.graphql b/static/service_schema.graphql index ddaf37f2..e69de29b 100644 --- a/static/service_schema.graphql +++ b/static/service_schema.graphql @@ -1,195 +0,0 @@ -""" -Changes that should be made to an instrument's configuration -""" -input ConfigurationUpdates { - """ - New template used to determine the root data directory - """ - directory: DirectoryTemplate - """ - New template used to determine the relative path to the main scan file for a collection - """ - scan: ScanTemplate - """ - New template used to determine the relative path for detector data files - """ - detector: DetectorTemplate - """ - The highest scan number to have been allocated. The next scan files generated will use the - next number. - """ - scanNumber: Int - """ - The extension of the files used to track scan numbers by GDA's numtracker facility - """ - trackerFileExtension: String -} - -""" -The current configuration for an instrument -""" -type CurrentConfiguration { - """ - The name of the instrument - """ - instrument: String! - """ - The template used to build the path to the data directory for an instrument - """ - directoryTemplate: String! - """ - The template used to build the path of a scan file for a data acquisition, relative to the - root of the data directory. - """ - scanTemplate: String! - """ - The template used to build the path of a detector's data file for a data acquisition, - relative to the root of the data directory. - """ - detectorTemplate: String! - """ - The latest scan number stored in the DB. This is the last scan number provided by this - service but may not reflect the most recent scan number for an instrument if an external - service (eg GDA) has incremented its own number tracker. - """ - dbScanNumber: Int! - """ - The highest matching number file for this instrument in the configured tracking directory. - May be null if no directory is available for this instrument or if there are no matching - number files. - """ - fileScanNumber: Int - """ - The file extension used for the file based tracking, eg using an extension of 'ext' - would create files `1.ext`, `2.ext` etc - """ - trackerFileExtension: String -} - -scalar Detector - -""" -GraphQL type to mimic a key-value pair from the map type that GraphQL doesn't have -""" -type DetectorPath { - """ - The name of the detector that should use this path - """ - name: String! - """ - The path where the detector should write its data - """ - path: String! -} - -""" -A template describing the location within a session data directory where the data for a given detector should be written - -It should contain placeholders for {detector} and {scan_number} to ensure paths are unique between scans and for multiple detectors. -""" -scalar DetectorTemplate - -""" -The path to a data directory and the components used to build it -""" -type DirectoryPath { - """ - The instrument session for which this is the data directory - """ - instrumentSession: String! - """ - The instrument for which this is the data directory - """ - instrument: String! - """ - The absolute path to the data directory - """ - path: String! -} - -""" -A template describing the path to the data directory for a given instrument session. It should be an absolute path and contain placeholders for {instrument} and {visit}. -""" -scalar DirectoryTemplate - -""" -Queries that modify the state of the numtracker configuration in some way -""" -type Mutation { - """ - Generate scan file locations for the next scan - """ - scan(instrument: String!, instrumentSession: String!, sub: Subdirectory): ScanPaths! - """ - Add or modify the stored configuration for an instrument - """ - configure(instrument: String!, config: ConfigurationUpdates!): CurrentConfiguration! -} - -""" -Queries relating to numtracker configurations that have no side-effects -""" -type Query { - """ - Get the data directory information for the given instrument and instrument session. - This information is not scan specific - """ - paths(instrument: String!, instrumentSession: String!): DirectoryPath! - """ - Get the current configuration for the given instrument - """ - configuration(instrument: String!): CurrentConfiguration! - """ - Get the configurations for all available instruments - Can be filtered to provide one or more specific instruments - """ - configurations(instrumentFilters: [String!]): [CurrentConfiguration!]! -} - -""" -Paths and values related to a specific scan/data collection for an instrument -""" -type ScanPaths { - """ - The directory used to generate this scan information. - """ - directory: DirectoryPath! - """ - The root scan file for this scan. The path has no extension so that the format can be - chosen by the client. - """ - scanFile: String! - """ - The scan number for this scan. This should be unique for the requested instrument. - """ - scanNumber: Int! - """ - The paths where the given detectors should write their files. - - Detector names are normalised before being used in file names by replacing any - non-alphanumeric characters with '_'. If there are duplicate names in the list - of detectors after this normalisation, there will be duplicate paths in the - results. - """ - detectors(names: [Detector!]!): [DetectorPath!]! -} - -""" -A template describing the location within a session data directory where the root scan file should be written. It should be a relative path and contain a placeholder for {scan_number} to ensure files are unique. -""" -scalar ScanTemplate - -scalar Subdirectory - -""" -Directs the executor to include this field or fragment only when the `if` argument is true. -""" -directive @include(if: Boolean!) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT -""" -Directs the executor to skip this field or fragment when the `if` argument is true. -""" -directive @skip(if: Boolean!) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT -schema { - query: Query - mutation: Mutation -} From 5d2ce32ad4540ba740cbb065c26d210abf30d8b6 Mon Sep 17 00:00:00 2001 From: Peter Holloway Date: Tue, 26 Aug 2025 17:41:10 +0100 Subject: [PATCH 5/7] Update schema --- static/service_schema.graphql | 208 ++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) diff --git a/static/service_schema.graphql b/static/service_schema.graphql index e69de29b..ef228ccd 100644 --- a/static/service_schema.graphql +++ b/static/service_schema.graphql @@ -0,0 +1,208 @@ +""" +Changes that should be made to an instrument's configuration +""" +input ConfigurationUpdates { + """ + New template used to determine the root data directory + """ + directory: DirectoryTemplate + """ + New template used to determine the relative path to the main scan file for a collection + """ + scan: ScanTemplate + """ + New template used to determine the relative path for detector data files + """ + detector: DetectorTemplate + """ + The highest scan number to have been allocated. The next scan files generated will use the + next number. + """ + scanNumber: Int + """ + The extension of the files used to track scan numbers by GDA's numtracker facility + """ + trackerFileExtension: String +} + +""" +The current configuration for an instrument +""" +type CurrentConfiguration { + """ + The name of the instrument + """ + instrument: String! + """ + The template used to build the path to the data directory for an instrument + """ + directoryTemplate: String! + """ + The template used to build the path of a scan file for a data acquisition, relative to the + root of the data directory. + """ + scanTemplate: String! + """ + The template used to build the path of a detector's data file for a data acquisition, + relative to the root of the data directory. + """ + detectorTemplate: String! + """ + The latest scan number stored in the DB. This is the last scan number provided by this + service but may not reflect the most recent scan number for an instrument if an external + service (eg GDA) has incremented its own number tracker. + """ + dbScanNumber: Int! + """ + The highest matching number file for this instrument in the configured tracking directory. + May be null if no directory is available for this instrument or if there are no matching + number files. + """ + fileScanNumber: Int + """ + The file extension used for the file based tracking, eg using an extension of 'ext' + would create files `1.ext`, `2.ext` etc + """ + trackerFileExtension: String +} + +scalar Detector + +""" +GraphQL type to mimic a key-value pair from the map type that GraphQL doesn't have +""" +type DetectorPath { + """ + The name of the detector that should use this path + """ + name: String! + """ + The path where the detector should write its data + """ + path: String! +} + +""" +A template describing the location within a session data directory where the data for a given detector should be written + +It should contain placeholders for {detector} and {scan_number} to ensure paths are unique between scans and for multiple detectors. +""" +scalar DetectorTemplate + +""" +The path to a data directory and the components used to build it +""" +type DirectoryPath { + """ + The instrument session for which this is the data directory + """ + instrumentSession: String! + """ + The instrument for which this is the data directory + """ + instrument: String! + """ + The absolute path to the data directory + """ + path: String! +} + +""" +A template describing the path to the data directory for a given instrument session. It should be an absolute path and contain placeholders for {instrument} and {visit}. +""" +scalar DirectoryTemplate + +""" +Queries that modify the state of the numtracker configuration in some way +""" +type Mutation { + """ + Generate scan file locations for the next scan + """ + scan(instrument: String!, instrumentSession: String!, sub: Subdirectory): ScanPaths! + """ + Add or modify the stored configuration for an instrument + """ + configure(instrument: String!, config: ConfigurationUpdates!): CurrentConfiguration! + registerTemplate(beamline: String!, template: TemplateInput!): NamedTemplate! +} + +type NamedTemplate { + name: String! + template: String! +} + +""" +Queries relating to numtracker configurations that have no side-effects +""" +type Query { + """ + Get the data directory information for the given instrument and instrument session. + This information is not scan specific + """ + paths(instrument: String!, instrumentSession: String!): DirectoryPath! + """ + Get the current configuration for the given instrument + """ + configuration(instrument: String!): CurrentConfiguration! + """ + Get the configurations for all available instruments + Can be filtered to provide one or more specific instruments + """ + configurations(instrumentFilters: [String!]): [CurrentConfiguration!]! + namedTemplates(beamline: String!, names: [String!]): [NamedTemplate!]! +} + +""" +Paths and values related to a specific scan/data collection for an instrument +""" +type ScanPaths { + """ + The directory used to generate this scan information. + """ + directory: DirectoryPath! + """ + The root scan file for this scan. The path has no extension so that the format can be + chosen by the client. + """ + scanFile: String! + """ + The scan number for this scan. This should be unique for the requested instrument. + """ + scanNumber: Int! + template(name: String!): String! + """ + The paths where the given detectors should write their files. + + Detector names are normalised before being used in file names by replacing any + non-alphanumeric characters with '_'. If there are duplicate names in the list + of detectors after this normalisation, there will be duplicate paths in the + results. + """ + detectors(names: [Detector!]!): [DetectorPath!]! +} + +""" +A template describing the location within a session data directory where the root scan file should be written. It should be a relative path and contain a placeholder for {scan_number} to ensure files are unique. +""" +scalar ScanTemplate + +scalar Subdirectory + +input TemplateInput { + name: String! + template: String! +} + +""" +Directs the executor to include this field or fragment only when the `if` argument is true. +""" +directive @include(if: Boolean!) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT +""" +Directs the executor to skip this field or fragment when the `if` argument is true. +""" +directive @skip(if: Boolean!) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT +schema { + query: Query + mutation: Mutation +} From 3b1fc53c4f4df6b8451e16c5f9fd8de64c9a45aa Mon Sep 17 00:00:00 2001 From: Peter Holloway Date: Wed, 27 Aug 2025 13:35:10 +0100 Subject: [PATCH 6/7] Raise errors instead of panicking --- src/graphql/mod.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/graphql/mod.rs b/src/graphql/mod.rs index a1ae4ae0..22ce02f2 100644 --- a/src/graphql/mod.rs +++ b/src/graphql/mod.rs @@ -228,7 +228,10 @@ impl ScanPaths { async fn template(&self, name: String) -> async_graphql::Result { Ok(path_to_string( - self.extra_templates.get(&name).unwrap().render(self), + self.extra_templates + .get(&name) + .ok_or(NoSuchTemplate(name))? + .render(self), )?) } @@ -316,6 +319,10 @@ struct TemplateInput { template: String, } +#[derive(Debug, Display, Error)] +#[display("Template {_0:?} not found")] +struct NoSuchTemplate(#[error(ignore)] String); + #[Object] impl NamedTemplate { async fn name(&self) -> &str { @@ -469,7 +476,7 @@ impl Mutation { .filter(|slct| slct.name() == "template") .flat_map(|slct| slct.arguments()) .filter_map(|args| { - args.get(0).map(|arg| { + args.first().map(|arg| { let Value::String(name) = &arg.1 else { panic!("name isn't a string") }; @@ -482,12 +489,9 @@ impl Mutation { .await? .into_iter() .map(|template| { - ( - template.name, - ScanTemplate::new_checked(&template.template).unwrap(), - ) + ScanTemplate::new_checked(&template.template).map(|tmpl| (template.name, tmpl)) }) - .collect(); + .collect::>()?; Ok(ScanPaths { directory: DirectoryPath { From d97b770ea67c6d64e006d9e429da9c1d21244c53 Mon Sep 17 00:00:00 2001 From: Peter Holloway Date: Wed, 27 Aug 2025 15:07:36 +0100 Subject: [PATCH 7/7] Update naming to match conventions --- src/graphql/mod.rs | 23 +++++++++++++---------- static/service_schema.graphql | 14 +++++++------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/graphql/mod.rs b/src/graphql/mod.rs index 22ce02f2..34e8ca2c 100644 --- a/src/graphql/mod.rs +++ b/src/graphql/mod.rs @@ -314,9 +314,9 @@ impl CurrentConfiguration { } #[derive(Debug, InputObject)] -struct TemplateInput { +struct NamedTemplateInput { name: String, - template: String, + template: InputTemplate, } #[derive(Debug, Display, Error)] @@ -421,17 +421,17 @@ impl Query { async fn named_templates<'ctx>( &self, ctx: &Context<'ctx>, - beamline: String, + instrument: String, names: Option>, ) -> async_graphql::Result> { check_auth(ctx, |policy, token| { - policy.check_instrument_admin(token, &beamline) + policy.check_instrument_admin(token, &instrument) }) .await?; let db = ctx.data::()?; match names { - Some(names) => Ok(db.additional_templates(&beamline, names).await?), - None => Ok(db.all_additional_templates(&beamline).await?), + Some(names) => Ok(db.additional_templates(&instrument, names).await?), + None => Ok(db.all_additional_templates(&instrument).await?), } } } @@ -530,13 +530,16 @@ impl Mutation { async fn register_template<'ctx>( &self, ctx: &Context<'ctx>, - beamline: String, - template: TemplateInput, + instrument: String, + template: NamedTemplateInput, ) -> async_graphql::Result { - check_auth(ctx, |pc, token| pc.check_instrument_admin(token, &beamline)).await?; + check_auth(ctx, |pc, token| { + pc.check_instrument_admin(token, &instrument) + }) + .await?; let db = ctx.data::()?; Ok(db - .register_template(&beamline, template.name, template.template) + .register_template(&instrument, template.name, template.template.to_string()) .await?) } } diff --git a/static/service_schema.graphql b/static/service_schema.graphql index ef228ccd..3cbbdc9b 100644 --- a/static/service_schema.graphql +++ b/static/service_schema.graphql @@ -124,7 +124,7 @@ type Mutation { Add or modify the stored configuration for an instrument """ configure(instrument: String!, config: ConfigurationUpdates!): CurrentConfiguration! - registerTemplate(beamline: String!, template: TemplateInput!): NamedTemplate! + registerTemplate(instrument: String!, template: NamedTemplateInput!): NamedTemplate! } type NamedTemplate { @@ -132,6 +132,11 @@ type NamedTemplate { template: String! } +input NamedTemplateInput { + name: String! + template: ScanTemplate! +} + """ Queries relating to numtracker configurations that have no side-effects """ @@ -150,7 +155,7 @@ type Query { Can be filtered to provide one or more specific instruments """ configurations(instrumentFilters: [String!]): [CurrentConfiguration!]! - namedTemplates(beamline: String!, names: [String!]): [NamedTemplate!]! + namedTemplates(instrument: String!, names: [String!]): [NamedTemplate!]! } """ @@ -189,11 +194,6 @@ scalar ScanTemplate scalar Subdirectory -input TemplateInput { - name: String! - template: String! -} - """ Directs the executor to include this field or fragment only when the `if` argument is true. """